feat: standalone testing check (#206)

* feat: standalone testing check

* feat: standalone testing check

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: sdk testing for business type

* feat: sdk testing for business type

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

* feat: update

.github/workflows/sdk-test.yaml

@@ -0,0 +1,62 @@
+name: Speakeasy Test SDK
+
+on:
+  workflow_call:
+    inputs:
+      target:
+        description: "The specific target to test"
+        required: false
+        type: string
+      working_directory:
+        description: "The working directory for running Speakeasy CLI commands in the action"
+        required: false
+        type: string
+    secrets:
+      github_access_token:
+        description: A GitHub access token with read access to the repo
+        required: true
+      speakeasy_api_key:
+        description: The API key to use to authenticate the Speakeasy CLI
+        required: true
+jobs:
+  test:
+    name: Test SDK
+    runs-on: ubuntu-latest
+    steps:
+      - name: Tune GitHub-hosted runner network
+        uses: smorimoto/tune-github-hosted-runner-network@v1
+
+      - name: Check commit message condition
+        id: check_commit
+        run: |
+          # The default trigger for this action will be a PR event.
+          # Sometimes we will also add a push event trigger to allow our Github app to run tests by making empty commits, a github token workaround.
+          # This check allows us to ensure we aren't double running tests when normal PR updates are made.
+          # The only time a push event should trigger tests is when it's a commit from our app with message "[run-tests]"
+          if [[ "${{ github.event_name }}" != "push" ]]; then
+            echo "Skipping commit message check since event is not push."
+            echo "run_tests=true" >> $GITHUB_OUTPUT
+            exit 0
+          fi
+
+          COMMIT_MESSAGE="${{ github.event.head_commit.message }}"
+          echo "Commit message: $COMMIT_MESSAGE"
+
+          if [[ "$COMMIT_MESSAGE" == *"[run-tests]"* ]]; then
+            echo "run_tests=true" >> $GITHUB_OUTPUT
+          else
+            echo "run_tests=false" >> $GITHUB_OUTPUT
+          fi
+
+      - id: test
+        name: Run Tests
+        if: steps.check_commit.outputs.run_tests == 'true'
+        uses: speakeasy-api/sdk-generation-action@v15
+        with:
+          action: "test"
+          working_directory: ${{ inputs.working_directory }}
+          target: ${{ inputs.target }}
+          speakeasy_api_key: ${{ secrets.speakeasy_api_key }}
+          github_access_token: ${{ secrets.github_access_token }}
+
+

.github/workflows/workflow-executor.yaml

@@ -109,6 +109,9 @@ on:
       speakeasy_api_key:
         description: The API key to use to authenticate the Speakeasy CLI
         required: true
+      pr_creation_pat:
+        description: A specific Github PAT used to create Pull Requests
+        required: false
       ossrh_username:
         description: A username for publishing the Java package to the OSSRH URL provided in gen.yml
         required: false
@@ -207,6 +210,7 @@ jobs:
           mode: ${{ inputs.mode }}
           force: ${{ inputs.force }}
           speakeasy_api_key: ${{ secrets.speakeasy_api_key }}
+          pr_creation_pat: ${{ secrets.pr_creation_pat }}
           output_tests: ${{ inputs.output_tests }}
           speakeasy_server_url: ${{ inputs.speakeasy_server_url }}
           working_directory: ${{ inputs.working_directory }}

action.yml

@@ -12,6 +12,9 @@ inputs:
   speakeasy_api_key:
     description: "The Speakeasy API key to authenticate the Speakeasy CLI with"
     required: true
+  pr_creation_pat:
+    description: "A specific Github PAT used to create Pull Requests"
+    required: false
   openapi_doc_auth_token:
     description: "An auth token to authenticate with a private OpenAPI spec"
     required: false
@@ -202,6 +205,7 @@ runs:
     SPEAKEASY_SERVER_URL: ${{ inputs.speakeasy_server_url }}
     OPENAI_API_KEY: ${{ inputs.openai_api_key }}
     OPENAPI_DOC_AUTH_TOKEN: ${{ inputs.openapi_doc_auth_token }}
+    PR_CREATION_PAT: ${{ inputs.pr_creation_pat }}
   args:
     - ${{ inputs.speakeasy_version }}
     - ${{ inputs.github_access_token }}

internal/actions/test.go

@@ -0,0 +1,93 @@
+package actions
+
+import (
+	"errors"
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	config "github.com/speakeasy-api/sdk-gen-config"
+	"github.com/speakeasy-api/sdk-generation-action/internal/cli"
+	"github.com/speakeasy-api/sdk-generation-action/internal/configuration"
+	"github.com/speakeasy-api/sdk-generation-action/internal/environment"
+	"golang.org/x/exp/slices"
+)
+
+func Test() error {
+	g, err := initAction()
+	if err != nil {
+		return err
+	}
+
+	if _, err = cli.Download("latest", g); err != nil {
+		return err
+	}
+
+	wf, err := configuration.GetWorkflowAndValidateLanguages(false)
+	if err != nil {
+		return err
+	}
+
+	// This will only come in via workflow dispatch, we do accept 'all' as a special case
+	var testedTargets []string
+	if providedTargetName := environment.SpecifiedTarget(); providedTargetName != "" {
+		testedTargets = append(testedTargets, providedTargetName)
+	}
+
+	if len(testedTargets) == 0 {
+		// We look for all files modified in the PR or Branch to see what SDK targets have been modified
+		files, err := g.GetChangedFilesForPRorBranch()
+		if err != nil {
+			fmt.Printf("Failed to get commited files: %s\n", err.Error())
+		}
+
+		for _, file := range files {
+			if strings.Contains(file, "gen.yaml") || strings.Contains(file, "gen.lock") {
+				cfgDir := filepath.Dir(file)
+				_, err := config.Load(filepath.Dir(file))
+				if err != nil {
+					return fmt.Errorf("failed to load config: %w", err)
+				}
+
+				outDir, err := filepath.Abs(filepath.Dir(cfgDir))
+				if err != nil {
+					return err
+				}
+				for name, target := range wf.Targets {
+					targetOutput := ""
+					if target.Output != nil {
+						targetOutput = *target.Output
+					}
+					targetOutput, err := filepath.Abs(filepath.Join(environment.GetWorkingDirectory(), targetOutput))
+					if err != nil {
+						return err
+					}
+					// If there are multiple SDKs in a workflow we ensure output path is unique
+					if targetOutput == outDir && !slices.Contains(testedTargets, name) {
+						testedTargets = append(testedTargets, name)
+					}
+				}
+			}
+		}
+	}
+	if len(testedTargets) == 0 {
+		fmt.Println("No target was provided ... skipping tests")
+		return nil
+	}
+
+	// we will pretty much never have a test action for multiple targets
+	// but if a customer manually setup their triggers in this way, we will run test sequentially for clear output
+	var errs []error
+	for _, target := range testedTargets {
+		// TODO: Once we have stable test reports we will probably want to use GH API to leave a PR comment/clean up old comments
+		if err := cli.Test(target); err != nil {
+			errs = append(errs, err)
+		}
+	}
+
+	if len(errs) > 0 {
+		return fmt.Errorf("test failures occured: %w", errors.Join(errs...))
+	}
+
+	return nil
+}

internal/cli/run.go

@@ -61,7 +61,8 @@ func Run(sourcesOnly bool, installationURLs map[string]string, repoURL string, r
 		args = append(args, "--set-version", environment.SetVersion())
 	}
 
-	if environment.SkipTesting() {
+	// If we are in PR mode we skip testing on generation, this should run as a PR check
+	if environment.SkipTesting() || (environment.GetMode() == environment.ModePR && !sourcesOnly) {
 		args = append(args, "--skip-testing")
 	}
 

internal/cli/speakeasy.go

@@ -262,3 +262,20 @@ func Tag(tags, sources, codeSamples []string) error {
 	fmt.Println(out)
 	return nil
 }
+
+func Test(target string) error {
+	args := []string{"test"}
+
+	if target != "all" {
+		args = append(args, "-t", target)
+	}
+
+	out, err := runSpeakeasyCommand(args...)
+	fmt.Println(out)
+	if err != nil {
+		fmt.Println(out)
+		return fmt.Errorf("error running speakeasy test for target %s: %w", target, err)
+	}
+
+	return nil
+}

internal/environment/environment.go

@@ -31,6 +31,7 @@ const (
 	ActionLog                Action = "log-result"
 	ActionPublishEvent       Action = "publish-event"
 	ActionTag                Action = "tag"
+	ActionTest               Action = "test"
 )
 
 const (

internal/git/git.go

@@ -576,12 +576,21 @@ Based on:
 		body = body[:maxBodyLength-3] + "..."
 	}
 
+	prClient := g.client
+	if providedPat := os.Getenv("PR_CREATION_PAT"); providedPat != "" {
+		ts := oauth2.StaticTokenSource(
+			&oauth2.Token{AccessToken: providedPat},
+		)
+		tc := oauth2.NewClient(context.Background(), ts)
+		prClient = github.NewClient(tc)
+	}
+
 	if info.PR != nil {
 		logging.Info("Updating PR")
 
 		info.PR.Body = github.String(body)
 		info.PR.Title = &title
-		info.PR, _, err = g.client.PullRequests.Edit(context.Background(), os.Getenv("GITHUB_REPOSITORY_OWNER"), getRepo(), info.PR.GetNumber(), info.PR)
+		info.PR, _, err = prClient.PullRequests.Edit(context.Background(), os.Getenv("GITHUB_REPOSITORY_OWNER"), getRepo(), info.PR.GetNumber(), info.PR)
 		// Set labels MUST always follow updating the PR
 		g.setPRLabels(context.Background(), os.Getenv("GITHUB_REPOSITORY_OWNER"), getRepo(), info.PR.GetNumber(), labelTypes, info.PR.Labels, labels)
 		if err != nil {
@@ -590,7 +599,7 @@ Based on:
 	} else {
 		logging.Info("Creating PR")
 
-		info.PR, _, err = g.client.PullRequests.Create(context.Background(), os.Getenv("GITHUB_REPOSITORY_OWNER"), getRepo(), &github.NewPullRequest{
+		info.PR, _, err = prClient.PullRequests.Create(context.Background(), os.Getenv("GITHUB_REPOSITORY_OWNER"), getRepo(), &github.NewPullRequest{
 			Title:               github.String(title),
 			Body:                github.String(body),
 			Head:                github.String(info.BranchName),
@@ -907,6 +916,116 @@ func getDownloadLinkFromReleases(releases []*github.RepositoryRelease, version s
 	return defaultDownloadUrl, defaultTagName
 }
 
+func (g *Git) GetChangedFilesForPRorBranch() ([]string, error) {
+	ctx := context.Background()
+	eventPath := os.Getenv("GITHUB_EVENT_PATH")
+	if eventPath == "" {
+		return nil, fmt.Errorf("no workflow event payload path")
+	}
+
+	data, err := os.ReadFile(eventPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read workflow event payload: %w", err)
+	}
+
+	var payload struct {
+		Number     int `json:"number"`
+		Repository struct {
+			DefaultBranch string `json:"default_branch"`
+		} `json:"repository"`
+	}
+
+	if err := json.Unmarshal(data, &payload); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal workflow event payload: %w", err)
+	}
+
+	prNumber := payload.Number
+	// This occurs if we come from a non-PR event trigger
+	if payload.Number == 0 {
+		ref := strings.TrimPrefix(environment.GetRef(), "refs/heads/")
+		if ref == "main" || ref == "master" {
+			// We just need to get the commit diff since we are not in a separate branch of PR
+			return g.GetCommitedFiles()
+		}
+		defaultBranch := "main"
+		if payload.Repository.DefaultBranch != "" {
+			fmt.Println("Default branch:", payload.Repository.DefaultBranch)
+			defaultBranch = payload.Repository.DefaultBranch
+		}
+
+		// Get the feature branch reference
+		branchRef, err := g.repo.Reference(plumbing.ReferenceName(environment.GetRef()), true)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get feature branch reference: %w", err)
+		}
+
+		// Get the latest commit on the feature branch
+		latestCommit, err := g.repo.CommitObject(branchRef.Hash())
+		if err != nil {
+			return nil, fmt.Errorf("failed to get latest commit of feature branch: %w", err)
+		}
+
+		var files []string
+		opt := &github.ListOptions{PerPage: 100} // Fetch 100 files per page (max: 300)
+		pageCount := 1                           // Track the number of API pages fetched
+
+		for {
+			comparison, resp, err := g.client.Repositories.CompareCommits(
+				ctx,
+				os.Getenv("GITHUB_REPOSITORY_OWNER"),
+				getRepo(),
+				defaultBranch,
+				latestCommit.Hash.String(),
+				opt,
+			)
+			if err != nil {
+				return nil, fmt.Errorf("failed to compare commits via GitHub API: %w", err)
+			}
+
+			// Collect filenames from this page
+			for _, file := range comparison.Files {
+				files = append(files, file.GetFilename())
+			}
+
+			// Check if there are more pages to fetch
+			if resp.NextPage == 0 {
+				break // No more pages, exit loop
+			}
+
+			opt.Page = resp.NextPage
+			pageCount++
+		}
+
+		logging.Info("Found %d files", len(files))
+		return files, nil
+
+	} else {
+		opts := &github.ListOptions{PerPage: 100}
+		var allFiles []string
+
+		// Fetch all changed files of the PR to determine testing coverage
+		for {
+			files, resp, err := g.client.PullRequests.ListFiles(ctx, os.Getenv("GITHUB_REPOSITORY_OWNER"), getRepo(), prNumber, opts)
+			if err != nil {
+				return nil, fmt.Errorf("failed to get changed files: %w", err)
+			}
+
+			for _, file := range files {
+				allFiles = append(allFiles, file.GetFilename())
+			}
+
+			if resp.NextPage == 0 {
+				break
+			}
+			opts.Page = resp.NextPage
+		}
+
+		logging.Info("Found %d files", len(allFiles))
+
+		return allFiles, nil
+	}
+}
+
 func (g *Git) GetCommitedFiles() ([]string, error) {
 	path := environment.GetWorkflowEventPayloadPath()
 

main.go

@@ -54,6 +54,8 @@ func main() {
 				return actions.Release()
 			case environment.ActionTag:
 				return actions.Tag()
+			case environment.ActionTest:
+				return actions.Test()
 			default:
 				return fmt.Errorf("unknown action: %s", environment.GetAction())
 			}