run corretness tests in docker

memgraph · Feb 12, 2025 · 5fc7b1d · 5fc7b1d
1 parent 12dba48
commit 5fc7b1d
Show file tree

Hide file tree

Showing 6 changed files with 53 additions and 66 deletions.
diff --git a/.github/workflows/reusable_test.yml b/.github/workflows/reusable_test.yml
@@ -39,6 +39,7 @@ env:
   MEMGRAPH_PORT: 7687
   NEO4J_PORT: 7688
   NEO4J_CONTAINER: "neo4j_test"
+  MEMGRAPH_NETWORK: "memgraph_test_network"
   OS: "ubuntu-24.04"
   s3_region: "eu-west-1"
   MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
@@ -110,41 +111,25 @@ jobs:
         if: env.MEMGRAPH_S3_URI
         run: aws s3 cp ${{ env.MEMGRAPH_S3_URI }} memgraph-${{ inputs.arch }}.deb
 
-      - name: Disk status before cleaning
-        run: |
-          df -h
-          docker buildx du
-
-      - name: Docker system prune
-        run: |
-          docker buildx prune --all -f
-
       - name: Build and run Memgraph MAGE:${{ inputs.build_target }}
         run: |
-
           DOCKER_BUILDKIT=1 docker buildx build \
           --tag memgraph-mage:${{ inputs.build_target }} \
           --target ${{ inputs.build_target }} \
           --platform linux/${{ inputs.arch }} \
           --file Dockerfile.${{ inputs.build_scope == 'without ML' && 'no_ML' || 'release' }} \
           --load .
+
+          docker network create ${{ env.MEMGRAPH_NETWORK }} || true
+
           docker run -d \
-            -p ${{ env.MEMGRAPH_PORT }}:7687 \
+            --rm \
+            --network ${{ env.MEMGRAPH_NETWORK }} \
             --name ${{ env.MAGE_CONTAINER }} \
             -e MEMGRAPH_ENTERPRISE_LICENSE=${{ env.MEMGRAPH_ENTERPRISE_LICENSE }} \
             -e MEMGRAPH_ORGANIZATION_NAME=${{ env.MEMGRAPH_ORGANIZATION_NAME }} \
             memgraph-mage:${{ inputs.build_target }} \
             --telemetry-enabled=False
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.PY_VERSION }}
-
-      - name: Install Python test dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r ./python/tests/requirements.txt --break-system-packages
       
       - name: Rust library tests
         if: inputs.build_target == 'dev'
@@ -164,7 +149,6 @@ jobs:
           docker exec -i -u root ${{ env.MAGE_CONTAINER }} bash -c "cd /mage/python/ && python3 -m pytest ."
 
       - name: Run End-to-end tests
-        if: inputs.arch != 'arm64'
         env:
           PYTHONPATH: "$PWD/e2e"
         run: |
@@ -173,8 +157,15 @@ jobs:
           "cd /mage/e2e/ && python3 -m pytest . -k 'not cugraph ${{ inputs.build_scope == 'without ML' && 'and not tgn and not node_classification and not link_prediction' || '' }}'"
 
       - name: Run End-to-end correctness tests
-        if: inputs.arch != 'arm64'
         env:
           PYTHONPATH: "$PWD/e2e"
         run: |
-          ./run_e2e_correctness_tests.sh ${{ env.MEMGRAPH_PORT }} ${{ env.NEO4J_PORT }} ${{ env.NEO4J_CONTAINER }}
+          ./run_e2e_correctness_tests.sh ${{ env.MEMGRAPH_PORT }} ${{ env.NEO4J_PORT }} ${{ env.NEO4J_CONTAINER }} ${{ env.MAGE_CONTAINER }} ${{ env.MEMGRAPH_NETWORK }}
+      
+      -name: Cleanup
+        run: |
+          docker stop ${{ env.MAGE_CONTAINER }}
+          docker rm ${{ env.MAGE_CONTAINER }}
+          docker rmi memgraph-mage:${{ inputs.build_target }}
+          docker network rm ${{ env.MEMGRAPH_NETWORK }}
+          docker stop ${{ env.NEO4J_CONTAINER }}
diff --git a/e2e_correctness/conftest.py b/e2e_correctness/conftest.py
@@ -1,3 +1,4 @@
 def pytest_addoption(parser):
     parser.addoption("--memgraph-port", type=int, action="store")
     parser.addoption("--neo4j-port", type=int, action="store")
+    parser.addoption("--neo4j-container", type=str, action="store")
diff --git a/e2e_correctness/query_neo_mem.py b/e2e_correctness/query_neo_mem.py
@@ -237,8 +237,8 @@ def create_graph_neo4j_json(json_neo4j_data) -> Graph:
     return graph
 
 
-def create_neo4j_driver(port: int) -> neo4j.BoltDriver:
-    return neo4j.GraphDatabase.driver(f"bolt://localhost:{port}", encrypted=False)
+def create_neo4j_driver(port: int, container: str) -> neo4j.BoltDriver:
+    return neo4j.GraphDatabase.driver(f"bolt://{container}:{port}", encrypted=False)
 
 
 def create_memgraph_db(port: int) -> gqlalchemy.Memgraph:

diff --git a/e2e_correctness/test_modules.py b/e2e_correctness/test_modules.py
@@ -27,9 +27,7 @@
     run_memgraph_query,
     run_neo4j_query,
     execute_query_neo4j,
-    path_to_string_neo4j,
     parse_neo4j,
-    path_to_string_mem,
     parse_mem,
 )
 
@@ -56,18 +54,6 @@ class TestConstants:
     MEMGRAPH_QUERY = "memgraph_query"
     NEO4J_QUERY = "neo4j_query"
     CONFIG_FILE = "config.yml"
-
-
-
-class ConfigConstants:
-    NEO4J_PORT = 7688
-    MEMGRAPH_PORT = 7687
-    NEO4J_CONTAINER_NAME = "test_neo4j_apoc"
-    MEMGRAPH_CONTAINER_NAME = "test_memgraph_mage"
-
-    NEO4J_IMAGE_NAME = "neo4j:latest"
-    MEMGRAPH_IMAGE_NAME = "memgraph-mage:test"
-
 
 def get_all_tests():
     """
@@ -253,10 +239,14 @@ def memgraph_db(memgraph_port):
 def neo4j_port(pytestconfig):
     return pytestconfig.getoption("--neo4j-port")
 
+@pytest.fixture(scope="session")
+def neo4j_container(pytestconfig):
+    return pytestconfig.getoption("--neo4j-container")
+
 
 @pytest.fixture(scope="session", autouse=True)
-def neo4j_driver(neo4j_port):
-    neo4j_driver = create_neo4j_driver(neo4j_port)
+def neo4j_driver(neo4j_port, neo4j_container):
+    neo4j_driver = create_neo4j_driver(neo4j_port, neo4j_container)
     logger.info("Created neo4j driver")
 
     yield neo4j_driver

diff --git a/run_e2e_correctness_tests.sh b/run_e2e_correctness_tests.sh
@@ -3,33 +3,26 @@
 MEMGRAPH_PORT=$1
 NEO4J_PORT=$2
 NEO4J_CONTAINER=$3
+MAGE_CONTAINER=$4
+MEMGRAPH_NETWORK=$5
 
-timeout=20
-counter=0
-
+echo "Start Neo4j..."
 docker run --rm \
-          --name "$NEO4J_CONTAINER"  \
-          -p 7474:7474 -p "$NEO4J_PORT":7687 \
-          -d \
-          -v "$HOME/neo4j/plugins:/plugins" \
-          --env NEO4J_AUTH=none  \
-          -e NEO4J_apoc_export_file_enabled=true \
-          -e NEO4J_apoc_import_file_enabled=true \
-          -e NEO4J_apoc_import_file_use__neo4j__config=true  \
-          -e NEO4J_PLUGINS='["apoc"]' neo4j:5.10.0
+    --name "$NEO4J_CONTAINER"  \
+    --network "$MEMGRAPH_NETWORK" \
+    -d \
+    -v "$HOME/neo4j/plugins:/plugins" \
+    --env NEO4J_AUTH=none  \
+    -e NEO4J_apoc_export_file_enabled=true \
+    -e NEO4J_apoc_import_file_enabled=true \
+    -e NEO4J_apoc_import_file_use__neo4j__config=true  \
+    -e NEO4J_PLUGINS='["apoc"]' neo4j:5.10.0
 
-echo "Waiting for Neo4j to start..."
-while ! curl --silent --fail http://localhost:7474; do
-  sleep 1
-  counter=$((counter+1))
-  if [ $counter -gt $timeout ]; then
-    echo "Neo4j failed to start in $timeout seconds"
-    exit 1
-  fi
-done
+echo "Installing python3 dependencies..."
+docker exec -i -u root "$MAGE_CONTAINER" bash -c "pip install -r /mage/python/tests/requirements.txt --break-system-packages"
 
-echo "Neo4j started. Running tests..."
-python3 test_e2e_correctness.py --memgraph-port "$MEMGRAPH_PORT" --neo4j-port "$NEO4J_PORT"
+echo "Running e2e correctness tests..."
+docker exec -i -u root "$MAGE_CONTAINER" bash -c "cd /mage/e2e_correctness && python3 test_e2e_correctness.py --memgraph-port $MEMGRAPH_PORT --neo4j-port $NEO4J_PORT --neo4j-container $NEO4J_CONTAINER"
 
 echo "Stopping Neo4j..."
 docker stop "$NEO4J_CONTAINER"
diff --git a/test_e2e_correctness.py b/test_e2e_correctness.py
@@ -12,6 +12,7 @@
 class ConfigConstants:
     NEO4J_PORT = 7688
     MEMGRAPH_PORT = 7687
+    NEO4J_CONTAINER_NAME = "neo4j"
 
 
 def parse_arguments():
@@ -34,6 +35,12 @@ def parse_arguments():
         type=int,
         required=False,
     )
+    parser.add_argument(
+        "--neo4j-container",
+        help="Set the Neo4j container name",
+        type=str,
+        required=False,
+    )
     args = parser.parse_args()
     return args
 
@@ -47,6 +54,7 @@ def main(
     test_filter: str = None,
     memgraph_port: str = str(ConfigConstants.MEMGRAPH_PORT),
     neo4j_port: str = str(ConfigConstants.NEO4J_PORT),
+    neo4j_container: str = ConfigConstants.NEO4J_CONTAINER_NAME,
 ):
     os.environ["PYTHONPATH"] = E2E_CORRECTNESS_DIRECTORY
     os.chdir(E2E_CORRECTNESS_DIRECTORY)
@@ -56,7 +64,7 @@ def main(
 
     command.extend(["--memgraph-port", memgraph_port])
     command.extend(["--neo4j-port", neo4j_port])
-
+    command.extend(["--neo4j-container", neo4j_container])
 
     try:
         subprocess.run(command, check=True)
@@ -70,14 +78,18 @@ def main(
     test_filter = args.k
     memgraph_port = args.memgraph_port
     neo4j_port = args.neo4j_port
+    neo4j_container = args.neo4j_container
 
     if memgraph_port:
         memgraph_port = str(memgraph_port)
     if neo4j_port:
         neo4j_port = str(neo4j_port)
+    if args.neo4j_container:
+        neo4j_container = args.neo4j_container
 
     main(
         test_filter=test_filter,
         memgraph_port=memgraph_port,
         neo4j_port=neo4j_port,
+        neo4j_container=neo4j_container,
     )