Upgrade Mage to Ubuntu 24.04 (#535)

.github/workflows/docker_publish.yml

@@ -89,8 +89,8 @@ jobs:
 
       - name: Download memgraph binary
         run: |
-          curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.memgraph_version }}/debian-11/memgraph_${{ github.event.inputs.memgraph_version }}-1_amd64.deb > memgraph-amd64.deb
-          curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.memgraph_version }}/debian-11-aarch64/memgraph_${{ github.event.inputs.memgraph_version }}-1_arm64.deb > memgraph-arm64.deb
+          curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.memgraph_version }}/ubuntu-24.04/memgraph_${{ github.event.inputs.memgraph_version }}-1_amd64.deb > memgraph-amd64.deb
+          curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.memgraph_version }}/ubuntu-24.04-aarch64/memgraph_${{ github.event.inputs.memgraph_version }}-1_arm64.deb > memgraph-arm64.deb
 
       - name: Build & push prod docker images
         run: |

.github/workflows/release_artifacts.yaml

@@ -5,11 +5,11 @@ on:
     inputs:
       os:
         type: choice
-        default: 'debian-11'
+        default: 'ubuntu-24.04'
         description: "Operating system to build the image for"
         required: true
         options:
-          - "debian-11"
+          - "ubuntu-24.04"
       mage_version:
         type: string
         description: "Mage version built into this image (format: X.Y.Z)"

.github/workflows/reusable_package_mage.yaml

@@ -60,7 +60,7 @@ on:
         default: false
 
 env:
-  OS: ${{ inputs.arch == 'arm' && 'debian-11-aarch64' || 'debian-11' }}
+  OS: ${{ inputs.arch == 'arm' && 'ubuntu-24.04-aarch64' || 'ubuntu-24.04' }}
   DOCKER_REPOSITORY_NAME: memgraph/memgraph-mage
 
 jobs:

.github/workflows/reusable_test.yml

@@ -33,13 +33,13 @@ on:
         default: true
 
 env:
-  PY_VERSION: "3.9"
+  PY_VERSION: "3.12"
   CORE_COUNT: "8"
   MAGE_CONTAINER: "mage"
   MEMGRAPH_PORT: 7687
   NEO4J_PORT: 7688
   NEO4J_CONTAINER: "neo4j_test"
-  OS: "debian-11"
+  OS: "ubuntu-24.04"
   s3_region: "eu-west-1"
   MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
   MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
@@ -159,7 +159,7 @@ jobs:
       - name: Install Python test dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r ./python/tests/requirements.txt
+          pip install -r ./python/tests/requirements.txt --break-system-packages
 
       - name: Build C++ modules
         run: |

.github/workflows/test.yml

@@ -69,7 +69,7 @@ jobs:
       build_scope: "${{ matrix.build_scope }}"
       memgraph_ref: "v${{ matrix.mg_version }}-${{ matrix.mg_rc_version }}"
       memgraph_ref_update: "false"
-      memgraph_download_link: "s3://deps.memgraph.io/memgraph/v${{ matrix.mg_version }}-${{ matrix.mg_rc_version }}/debian-11${{ matrix.arch == 'arm64' && '-aarch64' || '' }}${{ matrix.build_scope == 'dev' && '-relwithdebinfo' || '' }}/memgraph_${{ matrix.mg_version }}-1_${{ matrix.arch }}.deb"
+      memgraph_download_link: "s3://deps.memgraph.io/memgraph/v${{ matrix.mg_version }}-${{ matrix.mg_rc_version }}/ubuntu-24.04${{ matrix.arch == 'arm64' && '-aarch64' || '' }}${{ matrix.build_scope == 'dev' && '-relwithdebinfo' || '' }}/memgraph_${{ matrix.mg_version }}-1_${{ matrix.arch }}.deb"
     secrets: inherit
 
   PR_test:

Dockerfile.no_ML

@@ -1,14 +1,14 @@
-ARG PY_VERSION_DEFAULT=3.9
+ARG PY_VERSION_DEFAULT=3.12
 
-FROM debian:bullseye as base
+FROM ubuntu:24.04 as base
 
 USER root
 
 ARG TARGETARCH
 ARG PY_VERSION_DEFAULT
 ENV PY_VERSION ${PY_VERSION_DEFAULT}
 
-#essentials for production/dev
+# Essentials for production/dev
 RUN apt-get update && apt-get install -y \
     libcurl4        `memgraph` \
     libpython${PY_VERSION}   `memgraph` \
@@ -29,16 +29,14 @@ RUN apt-get update && apt-get install -y \
     uuid-dev \
     gdb \
     procps \
-    linux-perf \
     libc6-dbg \
     --no-install-recommends \
     && ln -s /usr/bin/$(ls /usr/bin | grep perf) /usr/bin/perf \
     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 COPY memgraph-${TARGETARCH}.deb .
 
-RUN dpkg -i memgraph-${TARGETARCH}.deb && rm memgraph-${TARGETARCH}.deb && rm /usr/lib/memgraph/query_modules/schema.so  \
-&& rm /usr/lib/memgraph/query_modules/example_cpp.so
+RUN dpkg -i memgraph-${TARGETARCH}.deb && rm memgraph-${TARGETARCH}.deb 
 
 ENV LD_LIBRARY_PATH /usr/lib/memgraph/query_modules
 
@@ -56,9 +54,9 @@ COPY . /mage
 #MAGE
 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y \
     && export PATH="/root/.cargo/bin:${PATH}" \
-    && python3 -m  pip install -r /mage/python/tests/requirements.txt \
-    && python3 -m  pip install -r /mage/python/requirements_no_ml.txt \
-    && python3 -m  pip install --no-cache-dir -r /usr/lib/memgraph/auth_module/requirements.txt \
+    && python3 -m  pip install -r /mage/python/tests/requirements.txt --break-system-packages \
+    && python3 -m  pip install -r /mage/python/requirements_no_ml.txt --break-system-packages \
+    && python3 -m  pip install --no-cache-dir -r /usr/lib/memgraph/auth_module/requirements.txt --break-system-packages \
     && python3 /mage/setup build -p /usr/lib/memgraph/query_modules/ --no-ml --cpp-build-flags CMAKE_BUILD_TYPE=${BUILD_TYPE} \
     && chown -R memgraph: /mage/e2e
 

Dockerfile.release

@@ -1,14 +1,15 @@
-ARG PY_VERSION_DEFAULT=3.9
+ARG PY_VERSION_DEFAULT=3.12
 
-FROM debian:bullseye as base
+FROM ubuntu:24.04 as base
 
 USER root
 
 ARG TARGETARCH
 ARG PY_VERSION_DEFAULT
 ENV PY_VERSION ${PY_VERSION_DEFAULT}
+ENV TORCH_VERSION=2.4.0
 
-#essentials for production/dev
+# Essentials for production/dev
 RUN apt-get update && apt-get install -y \
     libcurl4        `memgraph` \
     libpython${PY_VERSION}   `memgraph` \
@@ -29,7 +30,6 @@ RUN apt-get update && apt-get install -y \
     uuid-dev \
     gdb \
     procps \
-    linux-perf \
     libc6-dbg \
     --no-install-recommends \
     && ln -s /usr/bin/$(ls /usr/bin | grep perf) /usr/bin/perf \
@@ -54,45 +54,40 @@ ARG BUILD_TYPE=Release
 WORKDIR /mage
 COPY . /mage
 
+# First install requirements
+RUN python3 -m pip install -r /mage/python/requirements.txt --break-system-packages \
+    && python3 -m pip install -r /mage/python/tests/requirements.txt --break-system-packages \
+    && python3 -m pip install --no-cache-dir -r /usr/lib/memgraph/auth_module/requirements.txt --break-system-packages
 
-#MAGE
+# Torch and DGL 
+RUN python3 -m pip install torch-sparse torch-cluster torch-spline-conv torch-geometric torch-scatter -f https://data.pyg.org/whl/torch-2.3.0+cu121.html --break-system-packages \
+    && python3 -m pip install dgl -f https://data.dgl.ai/wheels/torch-2.3/cu121/repo.html --break-system-packages
+
+# Build query modules
 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y \
     && export PATH="/root/.cargo/bin:${PATH}" \
-    && python3 -m  pip install -r /mage/python/requirements.txt \
-    && python3 -m  pip install -r /mage/python/tests/requirements.txt \
-    && python3 -m  pip install --no-cache-dir -r /usr/lib/memgraph/auth_module/requirements.txt \
-    && python3 -m  pip install torch-sparse torch-cluster torch-spline-conv torch-geometric torch-scatter -f https://data.pyg.org/whl/torch-1.12.0+cu102.html \
     && python3 /mage/setup build -p /usr/lib/memgraph/query_modules/ --cpp-build-flags CMAKE_BUILD_TYPE=${BUILD_TYPE} \
     && chown -R memgraph: /mage/e2e
 
-#DGL build from source
-RUN git clone --recurse-submodules -b 0.9.x https://github.com/dmlc/dgl.git  \
-    && cd dgl && mkdir build && cd build && cmake .. \
-    && make -j4 && cd ../python \
-    && python3 setup.py bdist_wheel \
-    && pip3 install dist/*.whl
-
 USER memgraph
 ENTRYPOINT ["/usr/lib/memgraph/memgraph"]
 CMD [""]
 
-
-
 FROM base as prod
 
 USER root
 ENTRYPOINT []
 ARG PY_VERSION_DEFAULT
 ENV PY_VERSION ${PY_VERSION_DEFAULT}
 
-#copy modules
+# Copy modules
 COPY --from=dev /usr/lib/memgraph/query_modules/ /usr/lib/memgraph/query_modules/
 COPY --from=dev /usr/lib/memgraph/auth_module/ /usr/lib/memgraph/auth_module/
 
-#copy python build
+# Copy Python build
 COPY --from=dev /usr/local/lib/python${PY_VERSION}/ /usr/local/lib/python${PY_VERSION}/
 
-#copy e2e tests
+# Copy e2e tests
 COPY --from=dev /mage/e2e/ /mage/e2e/
 
 RUN mv /mage/e2e /e2e/ \
@@ -105,4 +100,4 @@ RUN mv /mage/e2e /e2e/ \
 
 USER memgraph
 ENTRYPOINT ["/usr/lib/memgraph/memgraph"]
-CMD [""]
+CMD [""]

README.md

@@ -158,14 +158,14 @@ git clone --recurse-submodules https://github.com/memgraph/mage.git && cd mage
 
 ```bash
 
-curl -L "https://download.memgraph.com/memgraph/v${MEMGRAPH_VERSION}/debian-11/memgraph_${MEMGRAPH_VERSION}-1_${ARCHITECTURE}.deb" > memgraph-${ARCHITECTURE}.deb
+curl -L "https://download.memgraph.com/memgraph/v${MEMGRAPH_VERSION}/ubuntu-24.04/memgraph_${MEMGRAPH_VERSION}-1_${ARCHITECTURE}.deb" > memgraph-${ARCHITECTURE}.deb
 
 ```
 
 or this one if you are on `arm64`:
 
 ```bash
-curl -L "https://download.memgraph.com/memgraph/v${MEMGRAPH_VERSION}/debian-11-aarch64/memgraph_${MEMGRAPH_VERSION}-1_arm64.deb" > memgraph-arm64.deb
+curl -L "https://download.memgraph.com/memgraph/v${MEMGRAPH_VERSION}/ubuntu-24.04-aarch64/memgraph_${MEMGRAPH_VERSION}-1_arm64.deb" > memgraph-arm64.deb
 ```
 
 **1.** To build the **MAGE** image run the following command where you set `${architecture}` to your system architecture (`amd64` or `arm64`):
@@ -207,7 +207,7 @@ To learn more about development with MAGE and Docker, visit the
   [site](https://memgraph.com/docs/memgraph/installation).
 - To build and install MAGE query modules you will need:
   - libcurl4
-  - libpython3.9
+  - libpython3.12
   - libssl-dev
   - libboost-all-dev
   - openssl 

e2e/kmeans_test/test_kmeans_small/test.yml

@@ -1,24 +1,24 @@
 query: >
-  CALL kmeans.get_clusters(2, "embedding", "k-means++", 10, 10, 0.0001, "auto", 1) YIELD node, cluster_id
+  CALL kmeans.get_clusters(2, "embedding", "k-means++", 10, 10, 0.0001, "lloyd", 1) YIELD node, cluster_id
   RETURN node.id as node_id, cluster_id
   ORDER BY node_id ASC;
 
 output:
   - node_id: 0
-    cluster_id: 1
+    cluster_id: 0
   - node_id: 1
-    cluster_id: 1
+    cluster_id: 0
   - node_id: 2
-    cluster_id: 1
+    cluster_id: 0
   - node_id: 4
-    cluster_id: 1
-  - node_id: 5
     cluster_id: 0
+  - node_id: 5
+    cluster_id: 1
   - node_id: 6
-    cluster_id: 0
+    cluster_id: 1
   - node_id: 7
-    cluster_id: 0
+    cluster_id: 1
   - node_id: 8
-    cluster_id: 0
+    cluster_id: 1
   - node_id: 9
-    cluster_id: 0
+    cluster_id: 1

python/kmeans.py

@@ -53,7 +53,7 @@ def get_clusters(
     n_init: mgp.Number = 10,
     max_iter: mgp.Number = 10,
     tol: mgp.Number = 1e-4,
-    algorithm: str = "auto",
+    algorithm: str = "lloyd",
     random_state: int = 1998,
 ) -> mgp.Record(node=mgp.Vertex, cluster_id=mgp.Number):
     nodes, embeddings = extract_nodes_embeddings(ctx, embedding_property)
@@ -85,7 +85,7 @@ def set_clusters(
     n_init: mgp.Number = 10,
     max_iter: mgp.Number = 10,
     tol: mgp.Number = 1e-4,
-    algorithm: str = "auto",
+    algorithm: str = "lloyd",
     random_state=1998,
 ) -> mgp.Record(node=mgp.Vertex, cluster_id=mgp.Number):
     nodes, embeddings = extract_nodes_embeddings(ctx, embedding_property)

python/requirements.txt

@@ -1,17 +1,17 @@
-gekko==0.2.8
-networkx==2.6.2
-python-Levenshtein==0.12.1
-elasticsearch==8.4.3
-gensim==4.3.3
-torch==1.12.0
-six==1.16.0
-torchmetrics==0.9.3
-igraph==0.10.2
-scikit-learn==0.24.2
-gqlalchemy==1.4.1
-mysql-connector-python==8.0.32
-oracledb==1.2.2
-pyodbc==4.0.35
-psycopg2-binary==2.9.9
 defusedxml==0.7.1
-scipy==1.12.0
+elasticsearch==8.17.0
+gekko==1.2.1
+gensim==4.3.3
+gqlalchemy==1.6.0
+igraph==0.11.8
+mysql-connector-python==9.1.0
+networkx==2.8.8
+oracledb==2.5.1
+psycopg2-binary==2.9.10
+pyodbc==5.2.0
+python-Levenshtein==0.26.1
+scikit-learn==1.5.2
+scipy==1.13.1
+six==1.17.0
+torch==2.3.0
+torchmetrics==0.10.0 # current implementation of node_classification module depends on version <= 0.10.0

python/requirements_no_ml.txt

@@ -1,15 +1,17 @@
-gekko==0.2.8
-networkx==2.6.2
-python-Levenshtein==0.12.1
-elasticsearch==8.4.3
-gensim==4.0.0
-six==1.16.0
-igraph==0.10.2
-scikit-learn==0.24.2
-gqlalchemy==1.4.1
-mysql-connector-python==8.0.32
-oracledb==1.2.2
-pyodbc==4.0.35
-psycopg2-binary==2.9.9
+cython==3.0.11
+numpy==1.26.0 # gekko 1.2.1 depends on numpy>=1.8.0 and gensim 4.3.3 depends on numpy<2.0 and >=1.18.5
 defusedxml==0.7.1
-scipy==1.12.0
+elasticsearch==8.17.0
+gekko==1.2.1
+gensim==4.3.3
+gqlalchemy==1.6.0 
+igraph==0.11.8
+mysql-connector-python==9.1.0
+networkx==2.8.8 # gqlalchemy 1.6.0 depends on networkx<3.0.0 and >=2.5.1
+oracledb==2.5.1
+psycopg2-binary==2.9.10
+pyodbc==5.2.0
+python-Levenshtein==0.26.1
+scikit-learn==1.6.1
+scipy==1.13.1 # gensim 4.3.3 depends on scipy<1.14.0 and >=1.7.0
+six==1.17.0

python/tests/requirements.txt

@@ -1,12 +1,12 @@
-pytest==7.0
-PyYAML==5.4.1
-black==22.3.0
-flake8==3.9.2
+pytest==8.3.4
+PyYAML==6.0.2
+black==24.10.0
+flake8==7.1.1
 pymgclient==1.3.1
 pytest-pylint==0.21.0
-pytest-timeout==1.4.2
-pytest-cov==2.12.1
-pytest-benchmark==3.4.1
-pytest-flake8==1.0.7
-pytest-black==0.3.12
-gqlalchemy==1.4.1
+pytest-timeout==2.3.1
+pytest-cov==6.0.0
+pytest-benchmark==5.1.0
+pytest-flake8==1.3.0
+pytest-black==0.6.0
+gqlalchemy==1.6.0