CLI: native collections support

.changeset/brave-islands-search.md

@@ -0,0 +1,5 @@
+---
+'@openfn/cli': minor
+---
+
+Support aliases on all project subcommands

.changeset/tiny-forks-brake.md

@@ -0,0 +1,5 @@
+---
+'@openfn/cli': minor
+---
+
+Auto-load collections adptor when using collections

.changeset/wicked-plants-push.md

@@ -0,0 +1,5 @@
+---
+'@openfn/lightning-mock': minor
+---
+
+Add basic collections support (GET only)

packages/cli/src/collections/command.ts

@@ -88,14 +88,6 @@ const key = {
   },
 };
 
-const token = {
-  name: 'pat',
-  yargs: {
-    alias: ['token'],
-    description: 'Lightning Personal Access Token (PAT)',
-  },
-};
-
 const endpoint = {
   name: 'endpoint',
   yargs: {
@@ -160,7 +152,7 @@ const updatedAfter = {
 const getOptions = [
   collectionName,
   key,
-  token,
+  o.apiKey,
   endpoint,
   pageSize,
   limit,
@@ -201,7 +193,7 @@ const dryRun = {
 const removeOptions = [
   collectionName,
   key,
-  token,
+  o.apiKey,
   endpoint,
   dryRun,
 
@@ -243,7 +235,7 @@ const setOptions = [
   override(key as any, {
     demand: false,
   }),
-  token,
+  o.apiKey,
   endpoint,
   value,
   items,

packages/cli/src/execute/command.ts

@@ -1,19 +1,22 @@
 import yargs from 'yargs';
-import { build, ensure } from '../util/command-builders';
+import { build, ensure, override } from '../util/command-builders';
 import * as o from '../options';
 
 import type { Opts } from '../options';
 
 export type ExecuteOptions = Required<
   Pick<
     Opts,
+    | 'apiKey'
     | 'adaptors'
     | 'autoinstall'
     | 'baseDir'
     | 'cacheSteps'
     | 'command'
     | 'compile'
     | 'credentials'
+    | 'collectionsEndpoint'
+    | 'collectionsVersion'
     | 'expandAdaptors'
     | 'end'
     | 'immutable'
@@ -44,10 +47,16 @@ const options = [
   o.expandAdaptors, // order is important
 
   o.adaptors,
+  override(o.apiKey, {
+    description: 'API token for collections',
+    alias: ['collections-api-key', 'collections-token', 'pat'],
+  }),
   o.autoinstall,
   o.cacheSteps,
   o.compile,
   o.credentials,
+  o.collectionsEndpoint,
+  o.collectionsVersion,
   o.end,
   o.ignoreImports,
   o.immutable,

packages/cli/src/options.ts

@@ -31,6 +31,8 @@ export type Opts = {
   configPath?: string;
   confirm?: boolean;
   credentials?: string;
+  collectionsEndpoint?: string;
+  collectionsVersion?: string;
   describe?: string;
   end?: string; // workflow end node
   expandAdaptors?: boolean; // for unit tests really
@@ -136,12 +138,17 @@ export const autoinstall: CLIOption = {
   },
 };
 
-export const apikey: CLIOption = {
+export const apiKey: CLIOption = {
   name: 'apikey',
   yargs: {
     alias: ['key', 'pat', 'token'],
     description:
-      '[beta only] API Key, Personal Access Token (Pat), or other access token',
+      'API Key, Personal Access Token (PAT), or other access token from Lightning',
+  },
+  ensure: (opts: any) => {
+    if (!opts.apikey) {
+      opts.apiKey = process.env.OPENFN_API_KEY;
+    }
   },
 };
 
@@ -240,6 +247,23 @@ export const configPath: CLIOption = {
   },
 };
 
+export const collectionsVersion: CLIOption = {
+  name: 'collections-version',
+  yargs: {
+    description:
+      'The version of the collections adaptor to use. Defaults to latest. Use OPENFN_COLLECTIONS_VERSION env.',
+  },
+};
+
+export const collectionsEndpoint: CLIOption = {
+  name: 'collections-endpoint',
+  yargs: {
+    alias: ['endpoint'],
+    description:
+      'The Lightning server to use for collections. Will use the project endpoint if available. Use OPENFN_COLLECTIONS_ENDPOINT env.',
+  },
+};
+
 export const credentials: CLIOption = {
   name: 'credentials',
   yargs: {

packages/cli/src/projects/fetch.ts

@@ -35,7 +35,7 @@ export type FetchOptions = Pick<
 
 const options = [
   po.alias,
-  o.apikey,
+  o.apiKey,
   o.endpoint,
   o.log,
   o.logJson,

packages/cli/src/projects/pull.ts

@@ -30,7 +30,7 @@ const options = [
   o2.workspace,
 
   // general options
-  o.apikey,
+  o.apiKey,
   o.endpoint,
   o.log,
   override(o.path, {

packages/cli/src/pull/command.ts

@@ -21,7 +21,7 @@ export type PullOptions = Required<
 >;
 
 const options = [
-  o.apikey,
+  o.apiKey,
   o.beta,
   o.configPath,
   o.endpoint,

packages/cli/src/util/load-plan.ts

@@ -331,11 +331,70 @@ const ensureAdaptors = (plan: CLIExecutionPlan) => {
   });
 };
 
+type ensureCollectionsOptions = {
+  endpoint?: string;
+  version?: string;
+  apiKey?: string;
+};
+
+const ensureCollections = (
+  plan: CLIExecutionPlan,
+  {
+    endpoint = 'https://app.openfn.org',
+    version = 'latest',
+    apiKey = 'null',
+  }: ensureCollectionsOptions = {},
+  logger?: Logger
+) => {
+  let collectionsFound = false;
+
+  Object.values(plan.workflow.steps)
+    .filter((step) => (step as any).expression?.match(/(collections\.)/))
+    .forEach((step) => {
+      const job = step as CLIJobNode;
+      if (
+        !job.adaptors?.find((v: string) =>
+          v.startsWith('@openfn/language-collections')
+        )
+      ) {
+        collectionsFound = true;
+        job.adaptors ??= [];
+        job.adaptors.push(
+          `@openfn/language-collections@${version || 'latest'}`
+        );
+
+        job.configuration = Object.assign({}, job.configuration, {
+          collections_endpoint: `${endpoint}/collections`,
+          collections_token: apiKey,
+        });
+      }
+    });
+
+  if (collectionsFound) {
+    if (!apiKey || apiKey === 'null') {
+      logger?.warn(
+        'WARNING: collections API was not set. Pass --api-key or OPENFN_API_KEY'
+      );
+    }
+    logger?.info(
+      `Configured collections to use endpoint ${endpoint} and API Key ending with ${apiKey?.substring(
+        apiKey.length - 10
+      )}`
+    );
+  }
+};
+
 const loadXPlan = async (
   plan: CLIExecutionPlan,
   options: Pick<
     Opts,
-    'monorepoPath' | 'baseDir' | 'expandAdaptors' | 'globals'
+    | 'monorepoPath'
+    | 'baseDir'
+    | 'expandAdaptors'
+    | 'globals'
+    | 'collectionsVersion'
+    | 'collectionsEndpoint'
+    | 'apiKey'
   >,
   logger: Logger,
   defaultName: string = ''
@@ -348,6 +407,15 @@ const loadXPlan = async (
     plan.workflow.name = defaultName;
   }
   ensureAdaptors(plan);
+  ensureCollections(
+    plan,
+    {
+      version: options.collectionsVersion,
+      apiKey: options.apiKey,
+      endpoint: options.collectionsEndpoint,
+    },
+    logger
+  );
 
   // import global functions
   // if globals is provided via cli argument. it takes precedence

packages/cli/test/util/load-plan.test.ts

@@ -4,7 +4,11 @@ import { createMockLogger } from '@openfn/logger';
 import type { Job } from '@openfn/lexicon';
 
 import loadPlan from '../../src/util/load-plan';
-import { Opts } from '../../src/options';
+import {
+  collectionsEndpoint,
+  collectionsVersion,
+  Opts,
+} from '../../src/options';
 
 const logger = createMockLogger(undefined, { level: 'debug' });
 
@@ -28,6 +32,7 @@ const createPlan = (steps: Partial<Job>[] = []) => ({
 test.beforeEach(() => {
   mock({
     'test/job.js': 'x',
+    'test/collections.js': 'collections.get()',
     'test/wf-old.json': JSON.stringify({
       start: 'a',
       jobs: [{ id: 'a', expression: 'x()' }],
@@ -114,6 +119,50 @@ test.serial('expression: set a start on the plan', async (t) => {
   t.is(plan.options.start, 'x');
 });
 
+test.serial('expression: load the collections adaptor', async (t) => {
+  const opts = {
+    expressionPath: 'test/collections.js',
+  } as Partial<Opts>;
+
+  const plan = await loadPlan(opts as Opts, logger);
+
+  t.deepEqual(plan.workflow.steps[0].adaptors, [
+    '@openfn/language-collections@latest',
+  ]);
+});
+
+test.serial(
+  'expression: load the collections adaptor with another',
+  async (t) => {
+    const opts = {
+      expressionPath: 'test/collections.js',
+      adaptors: ['@openfn/language-common@latest'],
+    } as Partial<Opts>;
+
+    const plan = await loadPlan(opts as Opts, logger);
+
+    t.deepEqual(plan.workflow.steps[0].adaptors, [
+      '@openfn/language-common@latest',
+      '@openfn/language-collections@latest',
+    ]);
+  }
+);
+test.serial(
+  'expression: load the collections adaptor with a specific version',
+  async (t) => {
+    const opts = {
+      expressionPath: 'test/collections.js',
+      collectionsVersion: '1.1.1',
+    } as Partial<Opts>;
+
+    const plan = await loadPlan(opts as Opts, logger);
+
+    t.deepEqual(plan.workflow.steps[0].adaptors, [
+      '@openfn/language-collections@1.1.1',
+    ]);
+  }
+);
+
 test.serial('xplan: load a plan from workflow path', async (t) => {
   const opts = {
     workflowPath: 'test/wf.json',
@@ -343,3 +392,40 @@ test.serial('xplan: support multiple adaptors', async (t) => {
   // @ts-ignore
   t.is(step.adaptor, undefined);
 });
+
+test.serial('xplan: append collections', async (t) => {
+  const opts = {
+    workflowPath: 'test/wf.json',
+    collectionsVersion: '1.1.1',
+    collectionsEndpoint: 'https://localhost:4000/',
+    apiKey: 'abc',
+  };
+
+  const plan = createPlan([
+    {
+      id: 'a',
+      expression: 'collections.get()',
+      adaptors: ['@openfn/language-common@1.0.0'],
+    },
+  ]);
+
+  mock({
+    'test/wf.json': JSON.stringify(plan),
+  });
+
+  const result = await loadPlan(opts, logger);
+  t.truthy(result);
+
+  const step = result.workflow.steps[0] as Job;
+  t.deepEqual(step.adaptors, [
+    '@openfn/language-common@1.0.0',
+    '@openfn/language-collections@1.1.1',
+  ]);
+  // @ts-ignore
+  t.is(step.adaptor, undefined);
+
+  t.deepEqual(step.configuration, {
+    collections_endpoint: `${opts.collectionsEndpoint}/collections`,
+    collections_token: opts.apiKey,
+  });
+});

packages/lightning-mock/package.json

@@ -18,6 +18,7 @@
   "dependencies": {
     "@koa/router": "^12.0.2",
     "@openfn/engine-multi": "workspace:*",
+    "@openfn/language-collections": "^0.6.2",
     "@openfn/lexicon": "workspace:^",
     "@openfn/logger": "workspace:*",
     "@openfn/runtime": "workspace:*",