Add ability to cache only latest N versions and cache github/gh-aw-actions (#136)

* add ability to cache only latest N versions

* fix test

config/actions/github_gh-aw-actions.json

@@ -0,0 +1,26 @@
+{
+  "owner": "github",
+  "repo": "gh-aw-actions",
+  "patterns": [
+    "+^master$",
+    "+^v[0-9]+(\\.[0-9]+){0,2}$"
+  ],
+  "branches": {},
+  "defaultBranch": "master",
+  "latestMajorVersions": 1,
+  "latestVersionsPerMajor": 3,
+  "tags": {
+    "v0.67.4": {
+      "commit": "2b3c275b3652caa01c2ebe31cbab50ec2df0f927",
+      "tag": "9d6ae06250fc0ec536a0e5f35de313b35bad7246"
+    },
+    "v0.68.0": {
+      "commit": "6715c81fe97e4bcbfa0734c3422491672ebda34f",
+      "tag": "0acfb4a691fe207cd8bc982ea5cb9d750d57a702"
+    },
+    "v0.68.1": {
+      "commit": "ea222e359276c0702a5f5203547ff9d88d0ddd76",
+      "tag": "2fe53acc038ba01c3bbdc767d4b25df31ca5bdfc"
+    }
+  }
+}

script/generated/github_gh-aw-actions.sh

@@ -0,0 +1,9 @@
+mkdir github_gh-aw-actions
+pushd github_gh-aw-actions
+curl -s -S -L -o '2b3c275b3652caa01c2ebe31cbab50ec2df0f927.tar.gz' 'https://api.github.com/repos/github/gh-aw-actions/tarball/2b3c275b3652caa01c2ebe31cbab50ec2df0f927'
+curl -s -S -L -o '2b3c275b3652caa01c2ebe31cbab50ec2df0f927.zip' 'https://api.github.com/repos/github/gh-aw-actions/zipball/2b3c275b3652caa01c2ebe31cbab50ec2df0f927'
+curl -s -S -L -o '6715c81fe97e4bcbfa0734c3422491672ebda34f.tar.gz' 'https://api.github.com/repos/github/gh-aw-actions/tarball/6715c81fe97e4bcbfa0734c3422491672ebda34f'
+curl -s -S -L -o '6715c81fe97e4bcbfa0734c3422491672ebda34f.zip' 'https://api.github.com/repos/github/gh-aw-actions/zipball/6715c81fe97e4bcbfa0734c3422491672ebda34f'
+curl -s -S -L -o 'ea222e359276c0702a5f5203547ff9d88d0ddd76.tar.gz' 'https://api.github.com/repos/github/gh-aw-actions/tarball/ea222e359276c0702a5f5203547ff9d88d0ddd76'
+curl -s -S -L -o 'ea222e359276c0702a5f5203547ff9d88d0ddd76.zip' 'https://api.github.com/repos/github/gh-aw-actions/zipball/ea222e359276c0702a5f5203547ff9d88d0ddd76'
+popd

script/internal/action-config.js

@@ -44,6 +44,18 @@ class ActionConfig {
    */
   defaultBranch = 'master'
 
+  /**
+   * Maximum number of latest major versions to include (default to unlimited)
+   * @type {number|undefined}
+   */
+  latestMajorVersions = undefined
+
+  /**
+   * Maximum number of latest version tags per major version (default to unlimited)
+   * @type {number|undefined}
+   */
+  latestVersionsPerMajor = undefined
+
   /**
    * Tag versions
    * @type {{[ref: string]: TagVersion}}
@@ -73,9 +85,11 @@ exports.TagVersion = TagVersion
  * @param {string[]} patternStrings
  * @param {string} defaultBranch
  * @param {string[]|undefined} ignoreTags
+ * @param {number|undefined} latestMajorVersions
+ * @param {number|undefined} latestVersionsPerMajor
  * @returns {Promise}
  */
-async function add(owner, repo, patternStrings, defaultBranch, ignoreTags) {
+async function add(owner, repo, patternStrings, defaultBranch, ignoreTags, latestMajorVersions, latestVersionsPerMajor) {
   assert.ok(owner, "Arg 'owner' must not be empty")
   assert.ok(repo, "Arg 'repo' must not be empty")
   assert.ok(patternStrings, "Arg 'patternStrings' must not be null")
@@ -94,6 +108,12 @@ async function add(owner, repo, patternStrings, defaultBranch, ignoreTags) {
   if (ignoreTags && ignoreTags.length > 0) {
     config.ignoreTags = ignoreTags
   }
+  if (latestMajorVersions && latestMajorVersions > 0) {
+    config.latestMajorVersions = latestMajorVersions
+  }
+  if (latestVersionsPerMajor && latestVersionsPerMajor > 0) {
+    config.latestVersionsPerMajor = latestVersionsPerMajor
+  }
   config.defaultBranch = defaultBranch
 
   const tempDir = path.join(paths.temp, `${owner}_${repo}`)
@@ -130,6 +150,9 @@ async function add(owner, repo, patternStrings, defaultBranch, ignoreTags) {
       config.tags[tag] = tagVersion
     }
 
+    // Prune old tags based on version limits
+    pruneOldTags(config)
+
     // Write config
     await exec.exec('mkdir', ['-p', path.dirname(file)])
     await fs.promises.writeFile(file, JSON.stringify(config, null, '  '))
@@ -141,6 +164,75 @@ async function add(owner, repo, patternStrings, defaultBranch, ignoreTags) {
 }
 exports.add = add
 
+/**
+ * Prunes old tags from the config based on latestMajorVersions and latestVersionsPerMajor.
+ * Modifies config.tags in place.
+ * @param {ActionConfig} config
+ */
+function pruneOldTags(config) {
+  const maxMajors = config.latestMajorVersions || 0
+  const maxPerMajor = config.latestVersionsPerMajor || 0
+
+  if (!maxMajors && !maxPerMajor) {
+    return
+  }
+
+  const tagNames = Object.keys(config.tags)
+  const versionTags = []
+  const keepTags = new Set()
+
+  for (const tag of tagNames) {
+    const match = tag.match(/^v(\d+)(?:\.(\d+))?(?:\.(\d+))?$/)
+    if (!match) {
+      // Always keep non-version tags
+      keepTags.add(tag)
+      continue
+    }
+
+    const major = parseInt(match[1], 10)
+    const minor = match[2] !== undefined ? parseInt(match[2], 10) : -1
+    const patch = match[3] !== undefined ? parseInt(match[3], 10) : -1
+    versionTags.push({ tag, major, minor, patch, isMajorOnly: minor === -1 })
+  }
+
+  // Distinct major versions sorted descending (newest first)
+  const majorVersions = [...new Set(versionTags.map(v => v.major))].sort((a, b) => b - a)
+  const allowedMajors = new Set(
+    maxMajors > 0 ? majorVersions.slice(0, maxMajors) : majorVersions
+  )
+
+  for (const major of allowedMajors) {
+    const tagsForMajor = versionTags.filter(v => v.major === major)
+
+    // Always keep major-only pointers (e.g. "v4")
+    for (const v of tagsForMajor.filter(v => v.isMajorOnly)) {
+      keepTags.add(v.tag)
+    }
+
+    // Sort non-major-only tags by version descending (latest first)
+    const sorted = tagsForMajor
+      .filter(v => !v.isMajorOnly)
+      .sort((a, b) => {
+        if (a.minor !== b.minor) return b.minor - a.minor
+        return b.patch - a.patch
+      })
+
+    const kept = maxPerMajor > 0 ? sorted.slice(0, maxPerMajor) : sorted
+    for (const v of kept) {
+      keepTags.add(v.tag)
+    }
+  }
+
+  // Remove pruned tags
+  for (const tag of tagNames) {
+    if (!keepTags.has(tag)) {
+      console.log(`Pruning tag '${tag}' from config (version limit)`)
+      delete config.tags[tag]
+    }
+  }
+}
+exports.pruneOldTags = pruneOldTags
+
 /**
  * Returns the action config file path
  * @param {string} owner

script/internal/add-action.js

@@ -15,6 +15,8 @@ async function main() {
     const patterns = args.patterns
     const defaultBranch = args.defaultBranch || 'master'
     const ignoreTags = args.ignoreTags
+    const latestMajorVersions = args.latestMajorVersions
+    const latestVersionsPerMajor = args.latestVersionsPerMajor
 
     // File exists?
     const file = actionConfig.getFilePath(owner, repo)
@@ -24,7 +26,7 @@ async function main() {
     await fsHelper.reinitTemp()
 
     // Add the config
-    await actionConfig.add(owner, repo, patterns, defaultBranch, ignoreTags)
+    await actionConfig.add(owner, repo, patterns, defaultBranch, ignoreTags, latestMajorVersions, latestVersionsPerMajor)
   }
   catch (err) {
     // Help
@@ -60,7 +62,7 @@ class Args {
  */
 function getArgs() {
   // Parse
-  const parsedArgs = argHelper.parse([], ['default-branch', 'ignore-tags'])
+  const parsedArgs = argHelper.parse([], ['default-branch', 'ignore-tags', 'latest-major-versions', 'latest-versions-per-major'])
   if (parsedArgs.arguments.length < 1) {
     argHelper.throwError('Expected at least one arg')
   }
@@ -101,16 +103,29 @@ function getArgs() {
     repo: splitNwo[1],
     patterns: patterns,
     defaultBranch: parsedArgs.options['default-branch'],
-    ignoreTags: ignoreTags
+    ignoreTags: ignoreTags,
+    latestMajorVersions: parseNonNegativeInt(parsedArgs.options['latest-major-versions'], 'latest-major-versions'),
+    latestVersionsPerMajor: parseNonNegativeInt(parsedArgs.options['latest-versions-per-major'], 'latest-versions-per-major')
   }
 }
 
+function parseNonNegativeInt(value, name) {
+  if (!value) return 0
+  const n = Number(value)
+  if (!Number.isInteger(n) || n < 0) {
+    argHelper.throwError(`--${name} must be a non-negative integer, got '${value}'`)
+  }
+  return n
+}
+
 function printUsage() {
-  console.error('USAGE: add-action.sh [--default-branch branch] [--ignore-tags versions] nwo [(+|-)regexp [...]]')
-  console.error(`  --default-branch  Default branch name. For example: master`)
-  console.error(`  --ignore-tags     Comma-separated version prefixes to ignore. For example: v1,v2`)
-  console.error(`  nwo               Name with owner. For example: actions/checkout`)
-  console.error(`  regexp            Refs to include or exclude. Default: ${actionConfig.defaultPatterns.join(' ')}`)
+  console.error('USAGE: add-action.sh [--default-branch branch] [--ignore-tags versions] [--latest-major-versions N] [--latest-versions-per-major N] nwo [(+|-)regexp [...]]')
+  console.error(`  --default-branch            Default branch name. For example: master`)
+  console.error(`  --ignore-tags               Comma-separated version prefixes to ignore. For example: v1,v2`)
+  console.error(`  --latest-major-versions     Only cache the latest N major versions. For example: 3`)
+  console.error(`  --latest-versions-per-major Only cache the latest N version tags per major version. For example: 5`)
+  console.error(`  nwo                         Name with owner. For example: actions/checkout`)
+  console.error(`  regexp                      Refs to include or exclude. Default: ${actionConfig.defaultPatterns.join(' ')}`)
 }
 
 main()

script/internal/filter-tags.js

@@ -0,0 +1,87 @@
+// Filters tags from an action config based on latestMajorVersions and latestVersionsPerMajor.
+// Reads JSON config from stdin, outputs allowed tag names (one per line).
+
+async function main() {
+  let input = ''
+  for await (const chunk of process.stdin) {
+    input += chunk
+  }
+
+  const config = JSON.parse(input)
+  const tags = Object.keys(config.tags || {})
+  const latestMajorVersions = config.latestMajorVersions || 0 // 0 = unlimited
+  const latestVersionsPerMajor = config.latestVersionsPerMajor || 0 // 0 = unlimited
+
+  if (!latestMajorVersions && !latestVersionsPerMajor) {
+    // No filtering configured, output all tags
+    for (const tag of tags) {
+      console.log(tag)
+    }
+    return
+  }
+
+  // Parse version info from tag names
+  const versionTags = []
+  const nonVersionTags = []
+
+  for (const tag of tags) {
+    const match = tag.match(/^v(\d+)(?:\.(\d+))?(?:\.(\d+))?$/)
+    if (!match) {
+      nonVersionTags.push(tag)
+      continue
+    }
+
+    const major = parseInt(match[1], 10)
+    const minor = match[2] !== undefined ? parseInt(match[2], 10) : -1
+    const patch = match[3] !== undefined ? parseInt(match[3], 10) : -1
+    const isMajorOnly = minor === -1
+
+    versionTags.push({ tag, major, minor, patch, isMajorOnly })
+  }
+
+  // Find distinct major versions sorted descending (newest first)
+  const majorVersions = [...new Set(versionTags.map(v => v.major))].sort((a, b) => b - a)
+
+  // Apply latestMajorVersions filter
+  const allowedMajors = new Set(
+    latestMajorVersions > 0 ? majorVersions.slice(0, latestMajorVersions) : majorVersions
+  )
+
+  // Always include non-version tags
+  const result = [...nonVersionTags]
+
+  for (const major of allowedMajors) {
+    const tagsForMajor = versionTags.filter(v => v.major === major)
+
+    // Always include major-only pointers (e.g., "v4")
+    for (const v of tagsForMajor.filter(v => v.isMajorOnly)) {
+      result.push(v.tag)
+    }
+
+    // Sort non-major-only tags by version descending (latest first)
+    const sortedVersions = tagsForMajor
+      .filter(v => !v.isMajorOnly)
+      .sort((a, b) => {
+        if (a.minor !== b.minor) return b.minor - a.minor
+        return b.patch - a.patch
+      })
+
+    // Apply latestVersionsPerMajor filter
+    const kept = latestVersionsPerMajor > 0
+      ? sortedVersions.slice(0, latestVersionsPerMajor)
+      : sortedVersions
+
+    for (const v of kept) {
+      result.push(v.tag)
+    }
+  }
+
+  for (const tag of result) {
+    console.log(tag)
+  }
+}
+
+main().catch(err => {
+  console.error(err.message)
+  process.exitCode = 1
+})

script/internal/generate-scripts.sh

@@ -44,6 +44,15 @@ for json_file in $script_dir/../../config/actions/*.json; do
   ignore_patterns=()
   IFS=$'\n' read -r -d '' -a ignore_patterns < <( echo "$json" | jq --raw-output '.ignoreTags // [] | .[]' && printf '\0' )
 
+  # Get version-filtered tags (applies latestMajorVersions and latestVersionsPerMajor)
+  filtered_tags=()
+  IFS=$'\n' read -r -d '' -a filtered_tags < <( echo "$json" | node "$script_dir/filter-tags.js" && printf '\0' )
+  unset filtered_tag_set
+  declare -A filtered_tag_set
+  for t in "${filtered_tags[@]}"; do
+    filtered_tag_set[$t]=1
+  done
+
   # Get an array of tag info. Each item contains "<tag> <commit_sha>"
   tag_info=()
   IFS=$'\n' read -r -d '' -a tag_info < <( echo "$json" | jq --raw-output '.tags | to_entries | .[] | .key + " " + .value.commit' && printf '\0' )
@@ -67,6 +76,12 @@ for json_file in $script_dir/../../config/actions/*.json; do
       continue
     fi
 
+    # Check if the tag passes version filter
+    if [ -z "${filtered_tag_set[$tag]+x}" ]; then
+      echo "Skipping tag '$tag' (filtered by version limits)"
+      continue
+    fi
+
     # Append curl download command
     curl_download_commands+=("curl -s -S -L -o '$sha.tar.gz' 'https://api.github.com/repos/$owner/$repo/tarball/$sha'")
     curl_download_commands+=("curl -s -S -L -o '$sha.zip' 'https://api.github.com/repos/$owner/$repo/zipball/$sha'")

script/internal/update-action.js

@@ -23,8 +23,10 @@ async function main() {
       const patterns = config.patterns
       const defaultBranch = config.defaultBranch
       const ignoreTags = config.ignoreTags
+      const latestMajorVersions = args.latestMajorVersions || config.latestMajorVersions
+      const latestVersionsPerMajor = args.latestVersionsPerMajor || config.latestVersionsPerMajor
       assert.ok(patterns && patterns.length, 'Existing patterns must not be empty')
-      await actionConfig.add(owner, repo, patterns, defaultBranch, ignoreTags)
+      await actionConfig.add(owner, repo, patterns, defaultBranch, ignoreTags, latestMajorVersions, latestVersionsPerMajor)
     }
   }
   catch (err) {
@@ -51,6 +53,8 @@ class Args {
   all = false
   owner = ''
   repo = ''
+  latestMajorVersions = 0
+  latestVersionsPerMajor = 0
 }
 
 /**
@@ -58,9 +62,11 @@ class Args {
  * @returns {Args}
  */
 function getArgs() {
-  const parsedArgs = argHelper.parse(['all'])
+  const parsedArgs = argHelper.parse(['all'], ['latest-major-versions', 'latest-versions-per-major'])
   const result = new Args()
   result.all = !!parsedArgs.flags['all']
+  result.latestMajorVersions = parseNonNegativeInt(parsedArgs.options['latest-major-versions'], 'latest-major-versions')
+  result.latestVersionsPerMajor = parseNonNegativeInt(parsedArgs.options['latest-versions-per-major'], 'latest-versions-per-major')
 
   // All
   if (result.all) {
@@ -88,9 +94,21 @@ function getArgs() {
   return result
 }
 
+function parseNonNegativeInt(value, name) {
+  if (!value) return 0
+  const n = Number(value)
+  if (!Number.isInteger(n) || n < 0) {
+    argHelper.throwError(`--${name} must be a non-negative integer, got '${value}'`)
+  }
+  return n
+}
+
 function printUsage() {
-  console.error('USAGE: update-action.sh nwo')
-  console.error(`  nwo     Name with owner. For example: actions/checkout`)
+  console.error('USAGE: update-action.sh [--all] [--latest-major-versions N] [--latest-versions-per-major N] [nwo]')
+  console.error(`  --all                       Update all configured actions`)
+  console.error(`  --latest-major-versions     Update to only keep the latest N major versions`)
+  console.error(`  --latest-versions-per-major Update to only keep the latest N version tags per major`)
+  console.error(`  nwo                         Name with owner. For example: actions/checkout`)
 }
 
 main()

script/test.sh

@@ -46,6 +46,8 @@ function test_tar_gz ()
         echo "Find action.yml under $sha_archive_full_path"
       elif [[ -f "$first_dir/action.yaml" ]]; then
         echo "Find action.yaml under $sha_archive_full_path"
+      elif find "$first_dir" -name 'action.yml' -o -name 'action.yaml' | grep -q .; then
+        echo "Find action.yml in subdirectory under $sha_archive_full_path"
       else
         echo "$sha_archive_full_path doesn't contain an action.yml or action.yaml"
         exit 1
@@ -83,6 +85,8 @@ function test_zip ()
         echo "Find action.yml under $sha_archive_full_path"
       elif [[ -f "$first_dir/action.yaml" ]]; then
         echo "Find action.yaml under $sha_archive_full_path"
+      elif find "$first_dir" -name 'action.yml' -o -name 'action.yaml' | grep -q .; then
+        echo "Find action.yml in subdirectory under $sha_archive_full_path"
       else
         echo "$sha_archive_full_path doesn't contain an action.yml or action.yaml"
         exit 1