From ca48fbf6d76996d71728126f84e44689be536168 Mon Sep 17 00:00:00 2001 From: Aldrian Harjati Date: Fri, 7 Feb 2025 11:33:13 -0500 Subject: [PATCH] update filer with run mode --- common/src/main/scala/hmda/util/Filer.scala | 63 +++++++++++++++---- common/src/test/resources/application.conf | 4 ++ .../src/test/scala/hmda/util/FilerSpec.scala | 32 ++++++++++ .../src/main/resources/application.conf | 2 + .../src/main/resources/application.conf | 2 + .../templates/deployment.yaml | 2 + kubernetes/hmda-data-publisher/values.yaml | 2 + .../templates/deployment.yaml | 2 + kubernetes/institutions-api/values.yaml | 3 + 9 files changed, 101 insertions(+), 11 deletions(-) create mode 100644 common/src/test/scala/hmda/util/FilerSpec.scala diff --git a/common/src/main/scala/hmda/util/Filer.scala b/common/src/main/scala/hmda/util/Filer.scala index 974bd0f954..74fde682a7 100644 --- a/common/src/main/scala/hmda/util/Filer.scala +++ b/common/src/main/scala/hmda/util/Filer.scala @@ -7,6 +7,7 @@ import java.time.temporal.{ ChronoField, TemporalAccessor } import com.typesafe.config.{ Config, ConfigFactory } import org.slf4j.LoggerFactory +import scala.util.matching.Regex import scala.util.Try object Filer { @@ -14,11 +15,51 @@ object Filer { private val dateFormatter = new DateTimeFormatterBuilder().appendPattern("MMMM dd yyyy").toFormatter - private val rtTgConfig = { - val tgWatch = ConfigFactory.load().getConfig("hmda.cm_watch.timed_guards") - val ns = Try(tgWatch.getString("ns")).getOrElse("default") - val cmName = Try(tgWatch.getString("name")).getOrElse("timed-guards") - new RealTimeConfig(cmName, ns) + private val config = ConfigFactory.load() + + private val runMode = config.getString("hmda.runtime.mode") + + private val rtTgConfigOpt: Option[RealTimeConfig] = { + if (runMode == "kubernetes") { + val tgWatch = config.getConfig("hmda.cm_watch.timed_guards") + val ns = Try(tgWatch.getString("ns")).getOrElse("default") + val cmName = Try(tgWatch.getString("name")).getOrElse("timed-guards") + log.info("Using real time configmap for timed guard.") + Some(new RealTimeConfig(cmName, ns)) + } else { + log.warn("Failed to load time guard through real time config.") + log.info("Using default for timed guard values.") + None + } + } + + /* + get default (stored in conf file) value from config key + */ + def getString(key: String): String = { + val QuarterReg: Regex = "q([1-3])(Start|End)".r + val ActionQuarterReg: Regex = "actionQ([1-3])(Start|End)".r + val quarterlyFiling: String = "hmda.rules.quarterly-filing" + val localKey: String = key match { + case "currentYear" => "hmda.filing.current" + case "yearsAllowed" => "hmda.rules.yearly-filing.years-allowed" + case "quarterlyYearsAllowed" => s"$quarterlyFiling.years-allowed" + case QuarterReg(q,t) => s"$quarterlyFiling.q$q.${t.toLowerCase()}" + case ActionQuarterReg(q,t) => s"$quarterlyFiling.q$q.action_date_${t.toLowerCase()}" + case _ => key + } + Try(config.getString(localKey)).getOrElse(localKey) + } + + /* + get configuration value from a key. if running in prod/dev, it should be using k8 + configmap. Else (for local/test), it should be using default config file ((stored in conf file) + */ + def getConfig(key: String): String = { + rtTgConfigOpt match { + case Some(rtTgConfig) => rtTgConfig.getString(key) + case _ => getString(key) + } } def check(filingRulesConfig: FilingRulesConfig)(year: Int, dayOfYear: Int, quarter: Option[String]): Boolean = { @@ -119,20 +160,20 @@ object Filer { private def getRulesFromRtConfig(): FilingRulesConfig = { val quarterlyFilingConfig = QuarterlyFilingConfig( - rtTgConfig.getString("quarterlyYearsAllowed").split(",").map(_.toInt).toList, + getConfig("quarterlyYearsAllowed").split(",").map(_.toInt).toList, getQuarterConfig(1), getQuarterConfig(2), getQuarterConfig(3) ) - FilingRulesConfig(quarterlyFilingConfig, rtTgConfig.getString("yearsAllowed").split(",").map(_.toInt).toList) + FilingRulesConfig(quarterlyFilingConfig, getConfig("yearsAllowed").split(",").map(_.toInt).toList) } private def getQuarterConfig(quarter: Int): QuarterConfig = { val currentYear = LocalDate.now().getYear - val startDate = rtTgConfig.getString(s"q${quarter}Start") - val endDate = rtTgConfig.getString(s"q${quarter}End") - val actionStartDate = rtTgConfig.getString(s"actionQ${quarter}Start") - val actionEndDate = rtTgConfig.getString(s"actionQ${quarter}End") + val startDate = getConfig(s"q${quarter}Start") + val endDate = getConfig(s"q${quarter}End") + val actionStartDate = getConfig(s"actionQ${quarter}Start") + val actionEndDate = getConfig(s"actionQ${quarter}End") QuarterConfig( dateFormatter.parse(s"$startDate $currentYear").get(ChronoField.DAY_OF_YEAR), dateFormatter.parse(s"$endDate $currentYear").get(ChronoField.DAY_OF_YEAR), diff --git a/common/src/test/resources/application.conf b/common/src/test/resources/application.conf index d8f91a4c65..ebc71109cd 100644 --- a/common/src/test/resources/application.conf +++ b/common/src/test/resources/application.conf @@ -6,4 +6,8 @@ akka { kafka { servers = "localhost:6001" +} + +hmda { + runtime.mode = "dev" } \ No newline at end of file diff --git a/common/src/test/scala/hmda/util/FilerSpec.scala b/common/src/test/scala/hmda/util/FilerSpec.scala new file mode 100644 index 0000000000..f9d40cbf28 --- /dev/null +++ b/common/src/test/scala/hmda/util/FilerSpec.scala @@ -0,0 +1,32 @@ +package hmda.util + +import hmda.util.Filer._ +import org.scalatest.{MustMatchers, PropSpec} +import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks + +class FilerSpec extends PropSpec with ScalaCheckPropertyChecks with MustMatchers { + + property("correctly get local config") { + // these values are from common/resource/reference.conf + getString("currentYear") mustBe "2021" + getString("yearsAllowed") mustBe "2018,2019,2020,2021,2022,2023,2024,2025" + getString("quarterlyYearsAllowed") mustBe "2020,2021,2022,2023,2024" + getString("q2Start") mustBe "July 01" + getString("q3End") mustBe "December 31" + getString("actionQ2Start") mustBe "April 01" + getString("actionQ3End") mustBe "September 30" + getString("randomKey") mustBe "randomKey" + } + + property("correctly get config") { + getConfig("currentYear") mustBe "2021" + getConfig("yearsAllowed") mustBe "2018,2019,2020,2021,2022,2023,2024,2025" + getConfig("quarterlyYearsAllowed") mustBe "2020,2021,2022,2023,2024" + getConfig("q2Start") mustBe "July 01" + getConfig("q3End") mustBe "December 31" + getConfig("actionQ2Start") mustBe "April 01" + getConfig("actionQ3End") mustBe "September 30" + getConfig("randomKey") mustBe "randomKey" + } + +} diff --git a/hmda-data-publisher/src/main/resources/application.conf b/hmda-data-publisher/src/main/resources/application.conf index d3f23b9874..d93eb8e251 100644 --- a/hmda-data-publisher/src/main/resources/application.conf +++ b/hmda-data-publisher/src/main/resources/application.conf @@ -111,6 +111,8 @@ hmda { validation.reportingUrl = "" validation.reportingUrl = ${?VALIDATION_REPORTING_URL} } + runtime.mode = "dev" + runtime.mode = ${?HMDA_RUNTIME_MODE} } private-aws { diff --git a/institutions-api/src/main/resources/application.conf b/institutions-api/src/main/resources/application.conf index 2bab333a10..6a250c7fa5 100644 --- a/institutions-api/src/main/resources/application.conf +++ b/institutions-api/src/main/resources/application.conf @@ -30,6 +30,8 @@ hmda { timeout = 5000 } } + runtime.mode = "dev" + runtime.mode = ${?HMDA_RUNTIME_MODE} } institution_db { diff --git a/kubernetes/hmda-data-publisher/templates/deployment.yaml b/kubernetes/hmda-data-publisher/templates/deployment.yaml index 93d9b6b3f1..b041f8ba02 100644 --- a/kubernetes/hmda-data-publisher/templates/deployment.yaml +++ b/kubernetes/hmda-data-publisher/templates/deployment.yaml @@ -44,6 +44,8 @@ spec: name: {{ include "hmda-data-publisher.name" . }}-dynamic-schedule-config optional: true env: + - name: HMDA_RUNTIME_MODE + value: { { .Values.hmda.runtimeMode } } - name: CASSANDRA_CLUSTER_HOSTS valueFrom: configMapKeyRef: diff --git a/kubernetes/hmda-data-publisher/values.yaml b/kubernetes/hmda-data-publisher/values.yaml index 44cf5f316a..efa2caedb4 100644 --- a/kubernetes/hmda-data-publisher/values.yaml +++ b/kubernetes/hmda-data-publisher/values.yaml @@ -23,6 +23,8 @@ grpc: targetPort: 60082 name: http2-publisher +hmda: + runtimeMode: kubernetes #ambassador: # name: ambassador-publisher # port: 80 diff --git a/kubernetes/institutions-api/templates/deployment.yaml b/kubernetes/institutions-api/templates/deployment.yaml index 8a5756c985..ccbd3603b7 100644 --- a/kubernetes/institutions-api/templates/deployment.yaml +++ b/kubernetes/institutions-api/templates/deployment.yaml @@ -43,6 +43,8 @@ spec: - configMapRef: name: {{ template "institutions-api.fullname" . }}-config env: + - name: HMDA_RUNTIME_MODE + value: {{ .Values.hmda.runtimeMode }} - name: KAFKA_INSTITUTIONS_TOPIC value: {{.Values.kafka.institutionsTopic}} - name: KAFKA_INSTITUTIONS_GROUP diff --git a/kubernetes/institutions-api/values.yaml b/kubernetes/institutions-api/values.yaml index cb45eb707e..02ea1ebc1c 100644 --- a/kubernetes/institutions-api/values.yaml +++ b/kubernetes/institutions-api/values.yaml @@ -23,6 +23,9 @@ kafka: institutionsGroup: institutions-group institutionsTopic: institution +hmda: + runtimeMode: kubernetes + #ambassador: # name: institutions-api-ambassador # service: