diff --git a/.changeset/brave-islands-search.md b/.changeset/brave-islands-search.md new file mode 100644 index 000000000..7e988d279 --- /dev/null +++ b/.changeset/brave-islands-search.md @@ -0,0 +1,5 @@ +--- +'@openfn/cli': minor +--- + +Support aliases on all project subcommands diff --git a/.changeset/tiny-forks-brake.md b/.changeset/tiny-forks-brake.md new file mode 100644 index 000000000..9d028192e --- /dev/null +++ b/.changeset/tiny-forks-brake.md @@ -0,0 +1,5 @@ +--- +'@openfn/cli': minor +--- + +Auto-load collections adptor when using collections diff --git a/.changeset/wicked-plants-push.md b/.changeset/wicked-plants-push.md new file mode 100644 index 000000000..a2ef97802 --- /dev/null +++ b/.changeset/wicked-plants-push.md @@ -0,0 +1,5 @@ +--- +'@openfn/lightning-mock': minor +--- + +Add basic collections support (GET only) diff --git a/packages/cli/src/collections/command.ts b/packages/cli/src/collections/command.ts index 4d872c1a8..4dd518ad0 100644 --- a/packages/cli/src/collections/command.ts +++ b/packages/cli/src/collections/command.ts @@ -88,14 +88,6 @@ const key = { }, }; -const token = { - name: 'pat', - yargs: { - alias: ['token'], - description: 'Lightning Personal Access Token (PAT)', - }, -}; - const endpoint = { name: 'endpoint', yargs: { @@ -160,7 +152,7 @@ const updatedAfter = { const getOptions = [ collectionName, key, - token, + o.apiKey, endpoint, pageSize, limit, @@ -201,7 +193,7 @@ const dryRun = { const removeOptions = [ collectionName, key, - token, + o.apiKey, endpoint, dryRun, @@ -243,7 +235,7 @@ const setOptions = [ override(key as any, { demand: false, }), - token, + o.apiKey, endpoint, value, items, diff --git a/packages/cli/src/execute/command.ts b/packages/cli/src/execute/command.ts index baa09401b..71cdc65cd 100644 --- a/packages/cli/src/execute/command.ts +++ b/packages/cli/src/execute/command.ts @@ -1,5 +1,5 @@ import yargs from 'yargs'; -import { build, ensure } from '../util/command-builders'; +import { build, ensure, override } from '../util/command-builders'; import * as o from '../options'; import type { Opts } from '../options'; @@ -7,6 +7,7 @@ import type { Opts } from '../options'; export type ExecuteOptions = Required< Pick< Opts, + | 'apiKey' | 'adaptors' | 'autoinstall' | 'baseDir' @@ -14,6 +15,8 @@ export type ExecuteOptions = Required< | 'command' | 'compile' | 'credentials' + | 'collectionsEndpoint' + | 'collectionsVersion' | 'expandAdaptors' | 'end' | 'immutable' @@ -44,10 +47,16 @@ const options = [ o.expandAdaptors, // order is important o.adaptors, + override(o.apiKey, { + description: 'API token for collections', + alias: ['collections-api-key', 'collections-token', 'pat'], + }), o.autoinstall, o.cacheSteps, o.compile, o.credentials, + o.collectionsEndpoint, + o.collectionsVersion, o.end, o.ignoreImports, o.immutable, diff --git a/packages/cli/src/options.ts b/packages/cli/src/options.ts index 901d8940f..3cd13869d 100644 --- a/packages/cli/src/options.ts +++ b/packages/cli/src/options.ts @@ -31,6 +31,8 @@ export type Opts = { configPath?: string; confirm?: boolean; credentials?: string; + collectionsEndpoint?: string; + collectionsVersion?: string; describe?: string; end?: string; // workflow end node expandAdaptors?: boolean; // for unit tests really @@ -136,12 +138,17 @@ export const autoinstall: CLIOption = { }, }; -export const apikey: CLIOption = { +export const apiKey: CLIOption = { name: 'apikey', yargs: { alias: ['key', 'pat', 'token'], description: - '[beta only] API Key, Personal Access Token (Pat), or other access token', + 'API Key, Personal Access Token (PAT), or other access token from Lightning', + }, + ensure: (opts: any) => { + if (!opts.apikey) { + opts.apiKey = process.env.OPENFN_API_KEY; + } }, }; @@ -240,6 +247,23 @@ export const configPath: CLIOption = { }, }; +export const collectionsVersion: CLIOption = { + name: 'collections-version', + yargs: { + description: + 'The version of the collections adaptor to use. Defaults to latest. Use OPENFN_COLLECTIONS_VERSION env.', + }, +}; + +export const collectionsEndpoint: CLIOption = { + name: 'collections-endpoint', + yargs: { + alias: ['endpoint'], + description: + 'The Lightning server to use for collections. Will use the project endpoint if available. Use OPENFN_COLLECTIONS_ENDPOINT env.', + }, +}; + export const credentials: CLIOption = { name: 'credentials', yargs: { diff --git a/packages/cli/src/projects/fetch.ts b/packages/cli/src/projects/fetch.ts index a5daa7b38..6358a2602 100644 --- a/packages/cli/src/projects/fetch.ts +++ b/packages/cli/src/projects/fetch.ts @@ -35,7 +35,7 @@ export type FetchOptions = Pick< const options = [ po.alias, - o.apikey, + o.apiKey, o.endpoint, o.log, o.logJson, diff --git a/packages/cli/src/projects/pull.ts b/packages/cli/src/projects/pull.ts index fd0decbdc..12bf8e6d8 100644 --- a/packages/cli/src/projects/pull.ts +++ b/packages/cli/src/projects/pull.ts @@ -30,7 +30,7 @@ const options = [ o2.workspace, // general options - o.apikey, + o.apiKey, o.endpoint, o.log, override(o.path, { diff --git a/packages/cli/src/pull/command.ts b/packages/cli/src/pull/command.ts index 3a7e6cc99..ba6dd6620 100644 --- a/packages/cli/src/pull/command.ts +++ b/packages/cli/src/pull/command.ts @@ -21,7 +21,7 @@ export type PullOptions = Required< >; const options = [ - o.apikey, + o.apiKey, o.beta, o.configPath, o.endpoint, diff --git a/packages/cli/src/util/load-plan.ts b/packages/cli/src/util/load-plan.ts index ff7137f2c..00dba22db 100644 --- a/packages/cli/src/util/load-plan.ts +++ b/packages/cli/src/util/load-plan.ts @@ -331,11 +331,70 @@ const ensureAdaptors = (plan: CLIExecutionPlan) => { }); }; +type ensureCollectionsOptions = { + endpoint?: string; + version?: string; + apiKey?: string; +}; + +const ensureCollections = ( + plan: CLIExecutionPlan, + { + endpoint = 'https://app.openfn.org', + version = 'latest', + apiKey = 'null', + }: ensureCollectionsOptions = {}, + logger?: Logger +) => { + let collectionsFound = false; + + Object.values(plan.workflow.steps) + .filter((step) => (step as any).expression?.match(/(collections\.)/)) + .forEach((step) => { + const job = step as CLIJobNode; + if ( + !job.adaptors?.find((v: string) => + v.startsWith('@openfn/language-collections') + ) + ) { + collectionsFound = true; + job.adaptors ??= []; + job.adaptors.push( + `@openfn/language-collections@${version || 'latest'}` + ); + + job.configuration = Object.assign({}, job.configuration, { + collections_endpoint: `${endpoint}/collections`, + collections_token: apiKey, + }); + } + }); + + if (collectionsFound) { + if (!apiKey || apiKey === 'null') { + logger?.warn( + 'WARNING: collections API was not set. Pass --api-key or OPENFN_API_KEY' + ); + } + logger?.info( + `Configured collections to use endpoint ${endpoint} and API Key ending with ${apiKey?.substring( + apiKey.length - 10 + )}` + ); + } +}; + const loadXPlan = async ( plan: CLIExecutionPlan, options: Pick< Opts, - 'monorepoPath' | 'baseDir' | 'expandAdaptors' | 'globals' + | 'monorepoPath' + | 'baseDir' + | 'expandAdaptors' + | 'globals' + | 'collectionsVersion' + | 'collectionsEndpoint' + | 'apiKey' >, logger: Logger, defaultName: string = '' @@ -348,6 +407,15 @@ const loadXPlan = async ( plan.workflow.name = defaultName; } ensureAdaptors(plan); + ensureCollections( + plan, + { + version: options.collectionsVersion, + apiKey: options.apiKey, + endpoint: options.collectionsEndpoint, + }, + logger + ); // import global functions // if globals is provided via cli argument. it takes precedence diff --git a/packages/cli/test/util/load-plan.test.ts b/packages/cli/test/util/load-plan.test.ts index 190dbb118..6e50dde63 100644 --- a/packages/cli/test/util/load-plan.test.ts +++ b/packages/cli/test/util/load-plan.test.ts @@ -4,7 +4,11 @@ import { createMockLogger } from '@openfn/logger'; import type { Job } from '@openfn/lexicon'; import loadPlan from '../../src/util/load-plan'; -import { Opts } from '../../src/options'; +import { + collectionsEndpoint, + collectionsVersion, + Opts, +} from '../../src/options'; const logger = createMockLogger(undefined, { level: 'debug' }); @@ -28,6 +32,7 @@ const createPlan = (steps: Partial[] = []) => ({ test.beforeEach(() => { mock({ 'test/job.js': 'x', + 'test/collections.js': 'collections.get()', 'test/wf-old.json': JSON.stringify({ start: 'a', jobs: [{ id: 'a', expression: 'x()' }], @@ -114,6 +119,50 @@ test.serial('expression: set a start on the plan', async (t) => { t.is(plan.options.start, 'x'); }); +test.serial('expression: load the collections adaptor', async (t) => { + const opts = { + expressionPath: 'test/collections.js', + } as Partial; + + const plan = await loadPlan(opts as Opts, logger); + + t.deepEqual(plan.workflow.steps[0].adaptors, [ + '@openfn/language-collections@latest', + ]); +}); + +test.serial( + 'expression: load the collections adaptor with another', + async (t) => { + const opts = { + expressionPath: 'test/collections.js', + adaptors: ['@openfn/language-common@latest'], + } as Partial; + + const plan = await loadPlan(opts as Opts, logger); + + t.deepEqual(plan.workflow.steps[0].adaptors, [ + '@openfn/language-common@latest', + '@openfn/language-collections@latest', + ]); + } +); +test.serial( + 'expression: load the collections adaptor with a specific version', + async (t) => { + const opts = { + expressionPath: 'test/collections.js', + collectionsVersion: '1.1.1', + } as Partial; + + const plan = await loadPlan(opts as Opts, logger); + + t.deepEqual(plan.workflow.steps[0].adaptors, [ + '@openfn/language-collections@1.1.1', + ]); + } +); + test.serial('xplan: load a plan from workflow path', async (t) => { const opts = { workflowPath: 'test/wf.json', @@ -343,3 +392,40 @@ test.serial('xplan: support multiple adaptors', async (t) => { // @ts-ignore t.is(step.adaptor, undefined); }); + +test.serial('xplan: append collections', async (t) => { + const opts = { + workflowPath: 'test/wf.json', + collectionsVersion: '1.1.1', + collectionsEndpoint: 'https://localhost:4000/', + apiKey: 'abc', + }; + + const plan = createPlan([ + { + id: 'a', + expression: 'collections.get()', + adaptors: ['@openfn/language-common@1.0.0'], + }, + ]); + + mock({ + 'test/wf.json': JSON.stringify(plan), + }); + + const result = await loadPlan(opts, logger); + t.truthy(result); + + const step = result.workflow.steps[0] as Job; + t.deepEqual(step.adaptors, [ + '@openfn/language-common@1.0.0', + '@openfn/language-collections@1.1.1', + ]); + // @ts-ignore + t.is(step.adaptor, undefined); + + t.deepEqual(step.configuration, { + collections_endpoint: `${opts.collectionsEndpoint}/collections`, + collections_token: opts.apiKey, + }); +}); diff --git a/packages/lightning-mock/package.json b/packages/lightning-mock/package.json index 997fee8f8..c212e2616 100644 --- a/packages/lightning-mock/package.json +++ b/packages/lightning-mock/package.json @@ -18,6 +18,7 @@ "dependencies": { "@koa/router": "^12.0.2", "@openfn/engine-multi": "workspace:*", + "@openfn/language-collections": "^0.6.2", "@openfn/lexicon": "workspace:^", "@openfn/logger": "workspace:*", "@openfn/runtime": "workspace:*", diff --git a/packages/lightning-mock/src/api-rest.ts b/packages/lightning-mock/src/api-rest.ts index 4411d8d8a..a7cffb522 100644 --- a/packages/lightning-mock/src/api-rest.ts +++ b/packages/lightning-mock/src/api-rest.ts @@ -84,7 +84,7 @@ workflows: `; export default ( - _app: DevServer, + app: DevServer, state: ServerState, _logger: Logger, _api: any @@ -115,5 +115,16 @@ export default ( ctx.response.status = 200; }); + router.get('/collections/:name/:key', (ctx) => { + const { name, key } = ctx.params; + try { + ctx.response.body = app.collections.fetch(name, key); + } catch (e: any) { + if ((e.message = 'COLLECTION_NOT_FOUND')) { + ctx.status = 404; + } + } + }); + return router.routes() as unknown as Koa.Middleware; }; diff --git a/packages/lightning-mock/src/server.ts b/packages/lightning-mock/src/server.ts index 2706282f7..22b6568a7 100644 --- a/packages/lightning-mock/src/server.ts +++ b/packages/lightning-mock/src/server.ts @@ -7,6 +7,7 @@ import createLogger, { LogLevel, Logger, } from '@openfn/logger'; +import { collections } from '@openfn/language-collections'; import type { StepId } from '@openfn/lexicon'; import type { LightningPlan, @@ -53,6 +54,9 @@ export type ServerState = { options: LightningOptions; projects: Record; + + /** Mock collections API (imported from the adaptor) */ + collections: any; }; export type LightningOptions = { @@ -76,6 +80,7 @@ const createLightningServer = (options: LightningOptions = {}) => { const runPrivateKey = options.runPrivateKey ? fromBase64(options.runPrivateKey) : undefined; + const state = { credentials: {}, runs: {}, @@ -98,6 +103,8 @@ const createLightningServer = (options: LightningOptions = {}) => { app.state = state; + app.collections = collections.createMockAPI(); + const port = options.port || 8888; const server = app.listen(port); logger.info('Listening on ', port); diff --git a/packages/lightning-mock/src/types.ts b/packages/lightning-mock/src/types.ts index cf81abec9..389ed7c2d 100644 --- a/packages/lightning-mock/src/types.ts +++ b/packages/lightning-mock/src/types.ts @@ -37,4 +37,7 @@ export type DevServer = Koa & { reset(): void; startRun(id: string): any; waitForResult(runId: string): Promise; + + /** Collections API (from the adaptor) */ + collections: any; }; diff --git a/packages/lightning-mock/test/rest.test.ts b/packages/lightning-mock/test/rest.test.ts index 421d2e08b..e3f89eb74 100644 --- a/packages/lightning-mock/test/rest.test.ts +++ b/packages/lightning-mock/test/rest.test.ts @@ -51,3 +51,20 @@ test.serial('should deploy a project and fetch it back', async (t) => { t.is(proj.id, 'abc'); t.is(proj.name, 'my project'); }); + +test.serial('should fetch items from a collection', async (t) => { + server.collections.createCollection('stuff'); + server.collections.upsert('stuff', 'x', { id: 'x' }); + + const response = await fetch(`${endpoint}/collections/stuff/*`); + const { items } = await response.json(); + t.is(items.length, 1); + t.deepEqual(items[0], { key: 'x', value: { id: 'x' } }); +}); + +test.serial("should return 404 if a collection isn't found", async (t) => { + const response = await fetch(`${endpoint}/collections/nope/*`); + t.is(response.status, 404); +}); + +test.todo("should return 403 if a collection isn't authorized");