3363 decommission kibana elasticsearch

.circleci/deployment/commands.yml

@@ -105,12 +105,6 @@
       frontend-appname:
         default: tdp-frontend
         type: string
-      kibana-appname:
-        default: tdp-kibana
-        type: string
-      proxy-appname:
-        default: tdp-elastic-proxy
-        type: string
       cf-space:
         default: tanf-dev
         type: string
@@ -134,8 +128,6 @@
               $DEPLOY_STRATEGY \
               <<parameters.frontend-appname>> \
               <<parameters.backend-appname>> \
-              <<parameters.kibana-appname>> \
-              <<parameters.proxy-appname>> \
               <<parameters.cf-space>>
 
   deploy-clamav:
@@ -159,9 +151,6 @@
       frontend-appname:
         default: tdp-frontend
         type: string
-      kibana-appname:
-        default: tdp-kibana
-        type: string
 # So the frontend knows what space its in for the banner.
 # I am unclear if the domain is a reliable metric to make this function
 # It seems like it might not be working
@@ -183,7 +172,6 @@
               $DEPLOY_STRATEGY \
               <<parameters.frontend-appname>> \
               <<parameters.backend-appname>> \
-              <<parameters.kibana-appname>> \
               <<parameters.cf-space>> \
               <<parameters.environment>>
 

.circleci/util/commands.yml

@@ -11,15 +11,6 @@
           name: Build and spin-up Django API service
           command: cd tdrs-backend; docker network create external-net; docker-compose up -d --build
 
-  docker-compose-up-with-elastic-backend:
-    steps:
-      - run:
-          name: Build and spin-up Django API service
-          command: cd tdrs-backend; docker network create external-net; docker-compose --profile elastic_setup up -d --build
-      - run:
-          name: Reindex elasticsearch
-          command: cd tdrs-backend; docker-compose exec web wait-for-it --service http://localhost:8080 --timeout 180 -- python manage.py search_index --rebuild -f
-
   cf-check:
     steps:
       - run:

scripts/deploy-backend.sh

@@ -10,9 +10,7 @@ DEPLOY_STRATEGY=${1}
 #The application name  defined via the manifest yml for the frontend
 CGAPPNAME_FRONTEND=${2}
 CGAPPNAME_BACKEND=${3}
-CGAPPNAME_KIBANA=${4}
-CGAPPNAME_PROXY=${5}
-CF_SPACE=${6}
+CF_SPACE=${4}
 
 strip() {
     # Usage: strip "string" "pattern"
@@ -22,14 +20,9 @@ strip() {
 env=$(strip $CF_SPACE "tanf-")
 backend_app_name=$(echo $CGAPPNAME_BACKEND | cut -d"-" -f3)
 
-# Update the Kibana and Elastic proxy names to include the environment
-CGAPPNAME_KIBANA="${CGAPPNAME_KIBANA}-${env}"
-CGAPPNAME_PROXY="${CGAPPNAME_PROXY}-${env}"
 
 echo DEPLOY_STRATEGY: "$DEPLOY_STRATEGY"
 echo BACKEND_HOST: "$CGAPPNAME_BACKEND"
-echo KIBANA_HOST: "$CGAPPNAME_KIBANA"
-echo ELASTIC_PROXY_HOST: "$CGAPPNAME_PROXY"
 echo CF_SPACE: "$CF_SPACE"
 echo env: "$env"
 echo backend_app_name: "$backend_app_name"
@@ -54,7 +47,6 @@ set_cf_envs()
   "DJANGO_SETTINGS_MODULE"
   "DJANGO_SU_NAME"
   "FRONTEND_BASE_URL"
-  "KIBANA_BASE_URL"
   "LOGGING_LEVEL"
   "REDIS_URI"
   "JWT_KEY"
@@ -96,18 +88,6 @@ generate_jwt_cert()
     cf set-env "$CGAPPNAME_BACKEND" JWT_KEY "$(cat key.pem)"
 }
 
-update_kibana()
-{
-  # Add network policy allowing Kibana to talk to the proxy and to allow the backend to talk to Kibana
-  cf add-network-policy "$CGAPPNAME_BACKEND" "$CGAPPNAME_KIBANA" --protocol tcp --port 5601
-  cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_KIBANA" --protocol tcp --port 5601
-  cf add-network-policy "$CGAPPNAME_KIBANA" "$CGAPPNAME_FRONTEND" --protocol tcp --port 80
-
-  # Upload dashboards to Kibana
-  CMD="curl -X POST $CGAPPNAME_KIBANA.apps.internal:5601/api/saved_objects/_import -H 'kbn-xsrf: true' --form file=@/home/vcap/app/tdpservice/search_indexes/kibana_saved_objs.ndjson"
-  cf run-task $CGAPPNAME_BACKEND --command "$CMD" --name kibana-obj-upload
-}
-
 prepare_promtail() {
   pushd tdrs-backend/plg/promtail
   CONFIG=config.yml
@@ -180,9 +160,6 @@ bind_backend_to_services() {
     cf bind-service "$CGAPPNAME_BACKEND" "tdp-datafiles-${env}"
     cf bind-service "$CGAPPNAME_BACKEND" "tdp-db-${env}"
 
-    # Setting up the ElasticSearch service
-    cf bind-service "$CGAPPNAME_BACKEND" "es-${env}"
-
     set_cf_envs
 
     echo "Restarting app: $CGAPPNAME_BACKEND"
@@ -225,8 +202,6 @@ else
   FRONTEND_BASE_URL="$DEFAULT_FRONTEND_ROUTE"
 fi
 
-KIBANA_BASE_URL="http://$CGAPPNAME_KIBANA.apps.internal"
-
 # Dynamically generate a new DJANGO_SECRET_KEY
 DJANGO_SECRET_KEY=$(python3 -c "from secrets import token_urlsafe; print(token_urlsafe(50))")
 
@@ -247,7 +222,6 @@ if [ "$DEPLOY_STRATEGY" = "rolling" ] ; then
     # Perform a rolling update for the backend and frontend deployments if
     # specified, otherwise perform a normal deployment
     update_backend 'rolling'
-    update_kibana 'rolling'
 elif [ "$DEPLOY_STRATEGY" = "bind" ] ; then
     # Bind the services the application depends on and restage the app.
     bind_backend_to_services
@@ -256,18 +230,15 @@ elif [ "$DEPLOY_STRATEGY" = "initial" ]; then
     # for it to work. the app will fail to start once, have the services bind,
     # and then get restaged.
     update_backend
-    update_kibana
     bind_backend_to_services
 elif [ "$DEPLOY_STRATEGY" = "rebuild" ]; then
     # You want to redeploy the instance under the same name
     # Delete the existing app (with out deleting the services)
     # and perform the initial deployment strategy.
     cf delete "$CGAPPNAME_BACKEND" -r -f
     update_backend
-    update_kibana
     bind_backend_to_services
 else
     # No changes to deployment config, just deploy the changes and restart
     update_backend
-    update_kibana
 fi

scripts/deploy-frontend.sh

@@ -8,21 +8,16 @@ DEPLOY_STRATEGY=${1}
 #The application name  defined via the manifest yml for the frontend
 CGHOSTNAME_FRONTEND=${2}
 CGHOSTNAME_BACKEND=${3}
-CGAPPNAME_KIBANA=${4}
-CF_SPACE=${5}
-ENVIRONMENT=${6}
+CF_SPACE=${4}
+ENVIRONMENT=${5}
 
 env=${CF_SPACE#"tanf-"}
 
-# Update the Kibana name to include the environment
-KIBANA_BASE_URL="${CGAPPNAME_KIBANA}-${env}.apps.internal"
-
 update_frontend()
 {
     echo DEPLOY_STRATEGY: "$DEPLOY_STRATEGY"
     echo FRONTEND_HOST: "$CGHOSTNAME_FRONTEND"
     echo BACKEND_HOST: "$CGHOSTNAME_BACKEND"
-    echo KIBANA_BASE_URL: "$KIBANA_BASE_URL"
     cd tdrs-frontend || exit
 
     if [ "$CF_SPACE" = "tanf-prod" ]; then
@@ -51,7 +46,6 @@ update_frontend()
     fi
 
     cf set-env "$CGHOSTNAME_FRONTEND" BACKEND_HOST "$CGHOSTNAME_BACKEND"
-    cf set-env "$CGHOSTNAME_FRONTEND" KIBANA_BASE_URL "$KIBANA_BASE_URL"
 
     npm run build:$ENVIRONMENT
     unlink .env.production

tdrs-backend/.env.example

@@ -81,8 +81,5 @@ AWS_REGION_NAME=us-gov-west-1
 POSTGRES_CHECK_TIMEOUT=30
 POSTGRES_CHECK_INTERVAL=1
 
-# Elastic stack
-ELASTIC_HOST=elastic:9200
-
 # testing
 CYPRESS_TOKEN=local-cypress-token

tdrs-backend/Pipfile

@@ -51,9 +51,6 @@ celery = "==5.2.6"
 redis = "==4.1.2"
 flower = "==1.1.0"
 django-celery-beat = "==2.2.1"
-elasticsearch = "==7.13.4" # REQUIRED - v7.14.0 introduces breaking changes
-django-elasticsearch-dsl = "==7.3"
-django-elasticsearch-dsl-drf = "==0.22.5"
 requests-aws4auth = "==1.1.2"
 cerberus = "==1.3.4"
 xlsxwriter = "==3.1.9"