Embeddings Refresher (#169)

* Adding Azure Function + updating deployment

* Make rules table a variable

* gitignoring bicep/main.json

* And removing it

* Oops, using the right file

* Rename

.github/bicep/main.bicep

@@ -5,6 +5,10 @@ param environment string = 'stage'
 param connectionString string
 @secure()
 param openAiApiKey string
+@secure()
+param GH_PAT string
+
+param rulesTableName string
 
 param allowedCors string
 param maxRequests string
@@ -26,7 +30,7 @@ var tenantId = subscription().tenantId
 var apiAppName = 'ssw-${appName}-api${prodEnvironmentName}'
 var frontendAppName = 'ssw-${appName}-webui${prodEnvironmentName}'
 var applicationInsightsName = 'ai-${appName}-${environment}'
-
+var functionAppName = 'func-${appName}-embeddings${prodEnvironmentName}'
 
 var lawName = 'laws-${appName}${prodEnvironmentName}'
 
@@ -44,12 +48,17 @@ resource storageAccount 'Microsoft.Storage/storageAccounts@2022-05-01' = {
   }
 }
 
+resource blobService 'Microsoft.Storage/storageAccounts/blobServices@2022-05-01' = {
+  parent: storageAccount
+  name: 'default'
+}
+
 resource hostingPlan 'Microsoft.Web/serverfarms@2021-03-01' existing = {
   name: hostingPlanName
   scope: resourceGroup(hostingPlanRgName)
 }
 
-resource kv 'Microsoft.KeyVault/vaults@2023-07-01' = {
+resource kv 'Microsoft.KeyVault/vaults@2022-07-01' = {
   name: keyVaultName
   location: location
   properties: {
@@ -71,22 +80,30 @@ resource kv 'Microsoft.KeyVault/vaults@2023-07-01' = {
   }
 }
 
-resource dbSecret 'Microsoft.KeyVault/vaults/secrets@2021-11-01-preview' = {
+resource dbSecret 'Microsoft.KeyVault/vaults/secrets@2022-07-01' = {
   parent: kv
   name: 'ConnectionStrings--DefaultConnection'
   properties: {
     value: connectionString
   }
 }
 
-resource openaiApiKeySecret 'Microsoft.KeyVault/vaults/secrets@2021-11-01-preview' = {
+resource openaiApiKeySecret 'Microsoft.KeyVault/vaults/secrets@2022-07-01' = {
   parent: kv
   name: 'OpenAiApiKey'
   properties: {
     value: openAiApiKey
   }
 }
 
+resource githubTokenSecret 'Microsoft.KeyVault/vaults/secrets@2022-07-01' = {
+  parent: kv
+  name: 'GithubToken'
+  properties: {
+    value: GH_PAT
+  }
+}
+
 resource backendAppService 'Microsoft.Web/sites@2020-12-01' = {
   name: apiAppName
   location: location
@@ -139,7 +156,71 @@ resource backendAppService 'Microsoft.Web/sites@2020-12-01' = {
   }
 }
 
-resource keyVaultAccessPolicy 'Microsoft.KeyVault/vaults/accessPolicies@2023-07-01' = {
+resource functionApp 'Microsoft.Web/sites@2022-03-01' = {
+  name: functionAppName
+  location: location
+  kind: 'functionapp,linux'
+  properties: {
+    serverFarmId: hostingPlan.id
+    siteConfig: {
+      linuxFxVersion: 'NODE|18'
+      appSettings: [
+        {
+          name: 'AzureWebJobsStorage'
+          value: 'DefaultEndpointsProtocol=https;AccountName=${storageAccount.name};EndpointSuffix=${az.environment().suffixes.storage};AccountKey=${storageAccount.listKeys().keys[0].value}'
+        }
+        {
+          name: 'FUNCTIONS_EXTENSION_VERSION'
+          value: '~4'
+        }
+        {
+          name: 'FUNCTIONS_WORKER_RUNTIME'
+          value: 'node'
+        }
+        {
+          name: 'WEBSITE_NODE_DEFAULT_VERSION'
+          value: '~18'
+        }
+        {
+          name: 'GITHUB_TOKEN'
+          value: '@Microsoft.KeyVault(SecretUri=${githubTokenSecret.properties.secretUri})'
+        }
+        {
+          name: 'OPENAI_API_KEY'
+          value: '@Microsoft.KeyVault(SecretUri=${openaiApiKeySecret.properties.secretUri})'
+        }
+        {
+          name: 'SUPABASE_KEY'
+          value: '@Microsoft.KeyVault(SecretUri=${dbSecret.properties.secretUri})'
+        }
+        {
+          name: 'GITHUB_OWNER'
+          value: 'SSWConsulting'
+        }
+        {
+          name: 'GITHUB_REPO'
+          value: 'SSW.Rules.Content'
+        }
+        {
+          name: 'APPINSIGHTS_INSTRUMENTATIONKEY'
+          value: applicationInsights.properties.InstrumentationKey
+        }
+        {
+          name: 'RULES_TABLE_NAME'
+          value: rulesTableName
+        }
+      ]
+      ftpsState: 'FtpsOnly'
+      minTlsVersion: '1.2'
+    }
+    httpsOnly: true
+  }
+  identity: {
+    type: 'SystemAssigned'
+  }
+}
+
+resource keyVaultAccessPolicy 'Microsoft.KeyVault/vaults/accessPolicies@2022-07-01' = {
   parent: kv
   name: 'add'
   properties: {
@@ -158,6 +239,20 @@ resource keyVaultAccessPolicy 'Microsoft.KeyVault/vaults/accessPolicies@2023-07-
           ]
         }
       }
+      {
+        objectId: functionApp.identity.principalId
+        tenantId: functionApp.identity.tenantId
+        permissions: {
+          secrets: [
+            'list'
+            'get'
+          ]
+          keys: [
+            'list'
+            'get'
+          ]
+        }
+      }
     ]
   }
 }

.github/workflows/main-build-deploy.yml

@@ -19,5 +19,7 @@ jobs:
 
       ConnectionString: ${{ secrets.CONNECTION_STRING }}
       OpenAiApiKey: ${{ secrets.OPENAI_API_KEY }}
+
+      GH_PAT: ${{ secrets.GH_PAT }}
 
       GithubRepoToken: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/stage-build-deploy.yml

@@ -23,5 +23,7 @@ jobs:
 
       ConnectionString: ${{ secrets.CONNECTION_STRING }}
       OpenAiApiKey: ${{ secrets.OPENAI_API_KEY }}
+
+      GH_PAT: ${{ secrets.GH_PAT }}
 
       GithubRepoToken: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/template-build-deploy.yml

@@ -54,7 +54,7 @@ jobs:
           subscriptionId: ${{ secrets.AZURE_SUBSCRIPTION }}
           resourceGroupName: ${{ secrets.AZURE_RG }}
           template: ./.github/bicep/main.bicep
-          parameters: 'appName=${{ inputs.appName }} environment=${{ inputs.deployEnvironment }} hostingPlanName=${{ vars.HOSTING_PLAN_NAME }} hostingPlanRgName=${{ vars.HOSTING_PLAN_RG_NAME }} connectionString=${{ secrets.ConnectionString }} openAiApiKey=${{ secrets.OpenAiApiKey }} allowedCors=${{ vars.ALLOWED_CORS }} maxRequests=${{ vars.MAX_REQUESTS }} signingAuthority=${{ vars.SIGNING_AUTHORITY }}'
+          parameters: 'appName=${{ inputs.appName }} environment=${{ inputs.deployEnvironment }} hostingPlanName=${{ vars.HOSTING_PLAN_NAME }} hostingPlanRgName=${{ vars.HOSTING_PLAN_RG_NAME }} connectionString=${{ secrets.ConnectionString }} openAiApiKey=${{ secrets.OpenAiApiKey }} GH_PAT=${{ secrets.GH_PAT }} allowedCors=${{ vars.ALLOWED_CORS }} maxRequests=${{ vars.MAX_REQUESTS }} signingAuthority=${{ vars.SIGNING_AUTHORITY }} rulesTableName=${{ vars.RULES_TABLE_NAME }}'
           failOnStdErr: false
 
       # TODO: Persist Bicep outputs to GH step outputs

.gitignore

@@ -368,4 +368,6 @@ FodyWeavers.xsd
 .idea/
 
 # Git
-.git/
+.git/
+
+.github/bicep/main.json

src/AzureFunctions/.funcignore

@@ -0,0 +1,10 @@
+*.js.map
+*.ts
+.git*
+.vscode
+local.settings.json
+test
+getting_started.md
+node_modules/@types/
+node_modules/azure-functions-core-tools/
+node_modules/typescript/

src/AzureFunctions/.gitignore

@@ -0,0 +1,48 @@
+bin
+obj
+csx
+.vs
+edge
+Publish
+
+*.user
+*.suo
+*.cscfg
+*.Cache
+project.lock.json
+
+/packages
+/TestResults
+
+/tools/NuGet.exe
+/App_Data
+/secrets
+/data
+.secrets
+appsettings.json
+local.settings.json
+
+node_modules
+dist
+
+# Local python packages
+.python_packages/
+
+# Python Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Azurite artifacts
+__blobstorage__
+__queuestorage__
+__azurite_db*__.json

src/AzureFunctions/.vscode/extensions.json

@@ -0,0 +1,5 @@
+{
+    "recommendations": [
+        "ms-azuretools.vscode-azurefunctions"
+    ]
+}

src/AzureFunctions/RefreshEmbeddings/function.json

@@ -0,0 +1,19 @@
+{
+  "bindings": [
+    {
+      "authLevel": "function",
+      "type": "httpTrigger",
+      "direction": "in",
+      "name": "req",
+      "methods": [
+        "get",
+        "post"
+      ]
+    },
+    {
+      "type": "http",
+      "direction": "out",
+      "name": "res"
+    }
+  ]
+}