Initial version of pull-through cache feature.

Closes #278

CHANGES/278.feature

@@ -0,0 +1 @@
+Added pull-through cache feature.

pulp_npm/app/models.py

@@ -1,10 +1,3 @@
-"""
-Check `Plugin Writer's Guide`_ for more details.
-
-.. _Plugin Writer's Guide:
-    http://docs.pulpproject.org/en/3.0/nightly/plugins/plugin-writer/index.html
-"""
-
 import json
 from logging import getLogger
 
@@ -21,7 +14,7 @@
 )
 
 from pulpcore.plugin.util import get_domain_pk
-from .utils import urlpath_sanitize
+from .utils import urlpath_sanitize, extract_package_info
 
 logger = getLogger(__name__)
 
@@ -32,16 +25,6 @@ class Package(Content):
 
     Define fields you need for your new content type and
     specify uniqueness constraint to identify unit of this type.
-
-    For example::
-
-        field1 = models.TextField()
-        field2 = models.IntegerField()
-        field3 = models.CharField()
-
-        class Meta:
-            default_related_name = "%(app_label)s_%(model_name)s"
-            unique_together = (field1, field2)
     """
 
     TYPE = "package"
@@ -58,6 +41,15 @@ def relative_path(self):
         """
         return f"{self.name}/-/{self.name}-{self.version}.tgz"
 
+    @staticmethod
+    def init_from_artifact_and_relative_path(artifact, relative_path):
+        name, version = extract_package_info(relative_path)
+
+        if not version:
+            return Package(name=name)
+
+        return Package(name=name, version=version)
+
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
         unique_together = ("name", "version", "_pulp_domain")
@@ -72,6 +64,15 @@ class NpmRemote(Remote):
 
     TYPE = "npm"
 
+    def get_remote_artifact_url(self, relative_path=None, request=None):
+        if request and (url := request.query.get("redirect")):
+            # This is a special case for pull-through caching
+            return url
+        return super().get_remote_artifact_url(relative_path, request=request)
+
+    def get_remote_artifact_content_type(self, relative_path=None):
+        return Package
+
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
 
@@ -105,6 +106,9 @@ class Meta:
     def content_handler(self, path):
         data = {}
 
+        if not self.repository:
+            return None
+
         repository_version = self.repository_version
         if not repository_version:
             repository_version = self.repository.latest_version()

pulp_npm/app/serializers.py

@@ -1,7 +1,8 @@
+from gettext import gettext as _
 from rest_framework import serializers
 
 from pulpcore.plugin import models as core_models
-from pulpcore.plugin import serializers as platform
+from pulpcore.plugin import serializers as core_serializers
 
 from . import models
 
@@ -12,7 +13,7 @@
 # If you want create content through upload, use "SingleArtifactContentUploadSerializer"
 # If you change this, make sure to do so on "fields" below, also.
 # Make sure your choice here matches up with the create() method of your viewset.
-class PackageSerializer(platform.SingleArtifactContentUploadSerializer):
+class PackageSerializer(core_serializers.SingleArtifactContentUploadSerializer):
     """
     A Serializer for Package.
 
@@ -26,7 +27,7 @@ class PackageSerializer(platform.SingleArtifactContentUploadSerializer):
     field3 = serializers.CharField()
 
     class Meta:
-        fields = platform.SingleArtifactContentSerializer.Meta.fields + (
+        fields = core_serializers.SingleArtifactContentSerializer.Meta.fields + (
             'field1', 'field2', 'field3'
         )
         model = models.Package
@@ -37,15 +38,15 @@ class Meta:
     relative_path = serializers.CharField()
 
     class Meta:
-        fields = platform.SingleArtifactContentUploadSerializer.Meta.fields + (
+        fields = core_serializers.SingleArtifactContentUploadSerializer.Meta.fields + (
             "name",
             "version",
             "relative_path",
         )
         model = models.Package
 
 
-class NpmRemoteSerializer(platform.RemoteSerializer):
+class NpmRemoteSerializer(core_serializers.RemoteSerializer):
     """
     A Serializer for NpmRemote.
 
@@ -56,9 +57,9 @@ class NpmRemoteSerializer(platform.RemoteSerializer):
     For example::
 
     class Meta:
-        validators = platform.RemoteSerializer.Meta.validators + [myValidator1, myValidator2]
+        validators = core_serializers.RemoteSerializer.Meta.validators + [myValidator1, ...]
 
-    By default the 'policy' field in platform.RemoteSerializer only validates the choice
+    By default the 'policy' field in core_serializers.RemoteSerializer only validates the choice
     'immediate'. To add on-demand support for more 'policy' options, e.g. 'streamed' or 'on_demand',
     re-define the 'policy' option as follows::
 
@@ -72,11 +73,11 @@ class Meta:
     )
 
     class Meta:
-        fields = platform.RemoteSerializer.Meta.fields
+        fields = core_serializers.RemoteSerializer.Meta.fields
         model = models.NpmRemote
 
 
-class NpmRepositorySerializer(platform.RepositorySerializer):
+class NpmRepositorySerializer(core_serializers.RepositorySerializer):
     """
     A Serializer for NpmRepository.
 
@@ -87,19 +88,27 @@ class NpmRepositorySerializer(platform.RepositorySerializer):
     For example::
 
     class Meta:
-        validators = platform.RepositorySerializer.Meta.validators + [myValidator1, myValidator2]
+        validators = core_serializers.RepositorySerializer.Meta.validators + [myValidator1, ...]
     """
 
     class Meta:
-        fields = platform.RepositorySerializer.Meta.fields
+        fields = core_serializers.RepositorySerializer.Meta.fields
         model = models.NpmRepository
 
 
-class NpmDistributionSerializer(platform.DistributionSerializer):
+class NpmDistributionSerializer(core_serializers.DistributionSerializer):
     """
     Serializer for NPM Distributions.
     """
 
+    remote = core_serializers.DetailRelatedField(
+        required=False,
+        help_text=_("Remote that can be used to fetch content when using pull-through caching."),
+        view_name_pattern=r"remotes(-.*/.*)?-detail",
+        queryset=core_models.Remote.objects.all(),
+        allow_null=True,
+    )
+
     class Meta:
-        fields = platform.DistributionSerializer.Meta.fields
+        fields = core_serializers.DistributionSerializer.Meta.fields + ("remote",)
         model = models.NpmDistribution

pulp_npm/app/utils.py

@@ -1,3 +1,6 @@
+import re
+
+
 def urlpath_sanitize(*args):
     """
     Join an arbitrary number of strings into a /-separated path.
@@ -15,3 +18,23 @@ def urlpath_sanitize(*args):
         if stripped:
             segments.append(stripped)
     return "/".join(segments)
+
+
+def extract_package_info(package_string):
+    """
+    Tries to extract the name and the version of a package
+    from the relative path string.
+
+    Args:
+        The relative_path string. "package/-/package-version.tgz"
+    """
+    pattern = r"(?P<name>[^/]+)(?:/-/(?P=name)-(?P<version>[^\.]+)\.\w+)?$"
+
+    match = re.match(pattern, package_string)
+
+    if match:
+        name = match.group("name")
+        version = match.group("version")
+        return name, version
+    else:
+        return None, None

pulp_npm/tests/functional/api/test_download_content.py

@@ -9,9 +9,7 @@
 
 from pulpcore.client.pulp_npm import RepositorySyncURL
 
-from pulp_npm.tests.functional.constants import (
-    NPM_FIXTURE_URL,
-)
+from pulp_npm.tests.functional.constants import NPM_FIXTURE_URL
 
 
 @pytest.mark.parallel

pulp_npm/tests/functional/api/test_pull_through_caching.py

@@ -0,0 +1,17 @@
+import pytest
+
+from urllib.parse import urljoin, urlsplit
+
+from pulp_npm.tests.functional.constants import NPM_FIXTURE_URL
+
+
+def test_pull_through_install(npm_bindings, npm_remote_factory, npm_distribution_factory, http_get):
+    """Test that a pull-through distro can be installed from."""
+    remote = npm_remote_factory(url=NPM_FIXTURE_URL)
+    distro = npm_distribution_factory(remote=remote.pulp_href)
+    PACKAGE = "commander"
+
+    __import__("ipdb").set_trace()
+    ye = http_get(f"{distro.base_url}/react")
+    # content = npm_bindings.ContentPackagesApi.list(name=PACKAGE)
+    # assert content.count == 1