diff --git a/extensions/.editorconfig b/extensions/.editorconfig index 484eef0..cbc1faf 100644 --- a/extensions/.editorconfig +++ b/extensions/.editorconfig @@ -1,5 +1,5 @@ # Default -[{.,common,tasks}/**.{ts,json}] +[{.,common,tasks,tabs}/**.{ts,json,html,css}] end_of_line = lf charset = utf-8 trim_trailing_whitespace = true @@ -8,7 +8,7 @@ indent_style = space indent_size = 4 # package.json typically uses 2 space indents -[{.,common,tasks}/**/package.json] +[{.,common,tasks,tabs}/**/package.json] end_of_line = lf charset = utf-8 trim_trailing_whitespace = true @@ -17,7 +17,7 @@ indent_style = space indent_size = 2 # Ignore pattern -[{.,common,tasks}/**/{node_modules/**,package-lock.json,yarn-lock.json}] +[{.,common,tasks,tabs}/**/{node_modules/**,package-lock.json,yarn-lock.json}] end_of_line = none charset = utf-8 trim_trailing_whitespace = none diff --git a/extensions/.gitignore b/extensions/.gitignore index c5217cc..54f3240 100644 --- a/extensions/.gitignore +++ b/extensions/.gitignore @@ -2,6 +2,9 @@ *.js node_modules +#include these script until TypeScript conversion +!/tabs/scripts/*.js + # Yarn / NPM yarn.lock package-lock.json diff --git a/extensions/package.json b/extensions/package.json index e3902ec..412e0ec 100644 --- a/extensions/package.json +++ b/extensions/package.json @@ -11,7 +11,8 @@ "lintFix": "npm run lintCommon -- --fix && npm run lintTasks -- --fix && npm run eclintFix", "buildCommon": "npm-recursive-install --rootDir=common && tsc -p common && npm pack common/nlu-devops-common/", "buildTasks": "npm-recursive-install --rootDir=tasks && tsc -p tasks", - "build": "npm run buildCommon && npm run buildTasks", + "buildTabs": "npm-recursive-install --rootDir=tabs", + "build": "npm run buildCommon && npm run buildTasks && npm run buildTabs", "postbuild": "npm run package", "package": "tfx extension create --rev-version --manifest-globs", "clean": "rimraf ./*.vsix && rimraf ./*.tgz", diff --git a/extensions/tabs/NLUResults.html b/extensions/tabs/NLUResults.html new file mode 100644 index 0000000..941723e --- /dev/null +++ b/extensions/tabs/NLUResults.html @@ -0,0 +1,46 @@ + + + + + + Confusion Matrix + + + + + + + + + + + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ + + diff --git a/extensions/tabs/README.md b/extensions/tabs/README.md new file mode 100644 index 0000000..e45af97 --- /dev/null +++ b/extensions/tabs/README.md @@ -0,0 +1,50 @@ +### Instructions + +1. In a console, run the command: `tsc -w` + + This will run the TypeScript compiler and have it constantly monitor for any changes to the *.tsc* files. + + Any changes made to those will automatically be propogated to the */dist* folder where the `.tsx` files are converted to `.js` and referenced by the `NLUResults.html`. + +2. In a console, run the command: `live-server .` + + This will run a dev server from the current location, open a new tab in your browser, and allow you to browse the pages + +3. Navigate to `nluresults.html` to view the charts + + +### How to debug in Chrome, real-time + +1. Follow the instructions above + +2. Create a `launch.json` file, per the instructions in the documentation. + +Mine looks like this: + +```json +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "chrome", + "request": "launch", + "name": "Launch Chrome against localhost", + "url": "http://localhost:8080/", + "webRoot": "${workspaceFolder}" + } + ] +} + +``` + +3. Read the **Attach** section of the [Code Debugger for Chrome docs](https://marketplace.visualstudio.com/items?itemName=msjsdiag.debugger-for-chrome) and attach the debugger to Chrome. + +3. Press *Debug* or **F5** to begin debugging. A new Chrome window will open. + +4. Set a breakpoint on any of the `.ts` files you are working on +---------------- + +**Written by:** Dave Voyles, Sept-12-2019 diff --git a/extensions/tabs/dist/confusion.js.map b/extensions/tabs/dist/confusion.js.map new file mode 100644 index 0000000..538e787 --- /dev/null +++ b/extensions/tabs/dist/confusion.js.map @@ -0,0 +1 @@ +{"version":3,"file":"confusion.js","sourceRoot":"","sources":["../src/confusion.ts"],"names":[],"mappings":"AAAA,4DAA4D;AAC5D,kCAAkC;AAElC,qDAAqD;AACrD,yCAAyC;AAEzC,mDAAmD;AAEnD,+BAA+B,QAAQ,EAAE,OAAO;IAC9C,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,QAAQ,EAAE,kBAAkB,EAAE,oBAAoB,CAAC;IACvD,IAAI,iBAAiB,EAAE,mBAAmB,CAAC;IAE3C,mDAAmD;IACnD,8DAA8D;IAC9D,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,OAAO,SAAA,EAAE,CAAC,CAAC,IAAI,CAAC,UAAS,IAAI;QAC/C,QAAQ,GAAG,EAAE,CAAC,IAAI,EAAE;aACjB,GAAG,CAAC,UAAS,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,UAAU,CAAA,CAAC,CAAC,CAAC;aACxC,OAAO,CAAC,IAAI,CAAC,CAAC;QACjB,oBAAoB,GAAG,EAAE,CAAC,IAAI,EAAE;aAC7B,GAAG,CAAC,UAAS,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,MAAM,CAAA,CAAC,CAAC,CAAC;aACtD,OAAO,CAAC,IAAI,CAAC,CAAC;QACjB,kBAAkB,GAAG,EAAE,CAAC,IAAI,EAAE;aAC3B,GAAG,CAAC,UAAS,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAA,CAAC,CAAC,CAAC;aACpD,OAAO,CAAC,IAAI,CAAC,CAAC;QAEjB,iBAAiB,GAAG,EAAE,CAAC,IAAI,EAAE;aACxB,GAAG,CAAC,UAAS,CAAC;YACX,IAAI,UAAU,GAAG,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC,UAAU,CAAC;YACvD,IAAI,QAAQ,GAAK,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC;YAE5C,uDAAuD;YACvD,EAAE,CAAC,CAAC,UAAU,IAAI,SAAS,CAAC,CAAC,CAAC;gBAC1B,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;oBACvD,IAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;oBAEhC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC;gBAC1B,CAAC;gBAAA,CAAC;YACN,CAAC;YAAC,IAAI,CAAC,CAAC;gBACJ,MAAM,CAAC,CAAC,iEAAiE;YAC7E,CAAC;QACL,CAAC,CAAC;aACD,OAAO,CAAC,IAAI,CAAC,CAAC;QAEnB,mBAAmB,GAAG,EAAE,CAAC,IAAI,EAAE;aAC1B,GAAG,CAAC,UAAS,CAAC;YACX,IAAI,QAAQ,GAAG,CAAC,CAAC,iBAAiB,CAAC,QAAQ,CAAC;YAE5C,EAAE,CAAC,CAAC,QAAQ,IAAI,SAAS,CAAC,CAAC,CAAC;gBACxB,uDAAuD;gBACvD,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;oBACnD,IAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;oBAEhC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC;gBAC9B,CAAC;gBAAA,CAAC;YACN,CAAC;YAAC,IAAI,CAAC,CAAC;gBACJ,MAAM,CAAC,CAAC,iEAAiE;YAC7E,CAAC;QACL,CAAC,CAAC;aACD,OAAO,CAAC,IAAI,CAAC,CAAC;QAEnB,GAAG,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;YAC/C,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;YAC3B,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC7E,CAAC;QAED,IAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAM,KAAK,GAAI,IAAI,KAAK,EAAE,CAAC;QAC3B,KAAK,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QAE1C,QAAQ,CAAC,KAAK,CAAC,CAAC;QAChB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QACvC,SAAS,CAAC,gBAAgB,EAAI,kBAAkB,CAAC,CAAC;QAClD,SAAS,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,CAAC;QAEpD,SAAS,CAAC,eAAe,EAAI,iBAAiB,CAAC,CAAC;QAChD,SAAS,CAAC,iBAAiB,EAAE,mBAAmB,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;AACL,CAAC;AAED,mBAAmB,OAAO,EAAE,IAAI;IAC9B,IAAI,SAAS,GAAG,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC;SACpB,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;SACxB,IAAI,CAAC,OAAO,CAAC,CAAC;IAEjB,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC;SACrB,IAAI,CAAC,IAAI,CAAC;SACV,KAAK,EAAE;SACP,MAAM,CAAC,KAAK,CAAC;SACb,IAAI,CAAC,UAAS,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACzC,CAAC;AAED,uBAAuB,IAAI,EAAE,MAAM,EAAE,KAAK;4BAC/B,KAAK;QACZ,IAAI,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC1C,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;aACxB,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACvB,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC;aACb,SAAS,CAAC,UAAU,CAAC;aACrB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;aACxB,KAAK,EAAE;aACP,MAAM,CAAC,QAAQ,CAAC;aAChB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;aACZ,IAAI,CAAC,IAAI,EAAE,UAAS,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,qBAAqB,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;aACjF,IAAI,CAAC,IAAI,EAAE,UAAS,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,qBAAqB,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;aACzE,IAAI,CAAC,QAAQ,EAAE,UAAS,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;aAC5D,IAAI,CAAC,MAAM,EAAE,UAAS,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;aAC1D,IAAI,CAAC,SAAS,EAAE,GAAG,CAAC;aACpB,EAAE,CAAC,WAAW,EAAE,UAAS,CAAC,EAAE,CAAC;YAC5B,EAAE,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACjD,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC;iBACb,MAAM,CAAC,QAAQ,CAAC;iBAChB,IAAI,CAAC,IAAI,EAAE,qBAAqB,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;iBACtD,IAAI,CAAC,IAAI,EAAE,qBAAqB,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;iBAC9C,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;iBACjC,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;iBAC/B,IAAI,CAAC,SAAS,EAAE,GAAG,CAAC;iBACpB,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;iBACvB,EAAE,CAAC,OAAO,EAAE;gBACX,EAAE,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;gBACnD,IAAM,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;gBACnC,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;gBACvC,EAAE,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YACnD,CAAC,CAAC;iBACD,EAAE,CAAC,WAAW,EAAE;gBACf,OAAO,CAAC,UAAU,EAAE;qBACjB,QAAQ,CAAC,GAAG,CAAC;qBACb,KAAK,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;gBACzB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,YAAY,CAAC;qBAClC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC;qBACtC,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;YAChD,CAAC,CAAC;iBACD,EAAE,CAAC,UAAU,EAAE;gBACd,OAAO,CAAC,UAAU,EAAE;qBACjB,QAAQ,CAAC,GAAG,CAAC;qBACb,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;gBACvB,EAAE,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACnD,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACP,CAAC;IA9CD,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE;gBAAvC,KAAK;KA8Cb;AACH,CAAC;AAED,kBAAkB,KAAK;IACrB,IAAM,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;IAE9B,IAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAE1C,IAAM,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;SACpD,IAAI,CAAC,SAAS,EAAE,OAAO,GAAG,KAAK,CAAC,UAAU,GAAG,GAAG,GAAG,KAAK,CAAC,WAAW,CAAC;SACrE,MAAM,CAAC,GAAG,CAAC;SACX,IAAI,CAAC,WAAW,EAAE,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,GAAG,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;IAEtF,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;SACf,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC;SACtB,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC;SAC/B,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;IAErC,IAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC;SACtB,IAAI,CAAC,WAAW,EAAE,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,GAAG,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;IAExF,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACV,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC;SACvB,IAAI,CAAC,WAAW,EAAE,gBAAgB,CAAC;SACnC,IAAI,CAAC,KAAK,CAAC,CAAC;IAEf,CAAC,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;SACtD,KAAK,EAAE;SACP,MAAM,CAAC,MAAM,CAAC;SACd,IAAI,CAAC,OAAO,EAAE,gBAAgB,CAAC;SAC/B,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;SACb,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC;SACvB,IAAI,CAAC,IAAI,EAAE,UAAS,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;SAC7C,IAAI,CAAC,IAAI,EAAE,UAAS,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;SAC7C,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC;SACpB,IAAI,CAAC,iBAAiB,EAAE,YAAY,CAAC;SACrC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC;SAC3B,IAAI,CAAC,kBAAkB,EAAE,OAAO,CAAC;SACjC,IAAI,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;IAE/B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;SACb,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;SACb,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;SACb,IAAI,CAAC,eAAe,CAAC,CAAC;IACzB,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;SACb,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC;SAC5B,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;SACb,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC1B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;SACb,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;SACb,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;SAC3B,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC1B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;SACb,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC;SAC5B,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;SAC3B,IAAI,CAAC,eAAe,CAAC,CAAC;IACzB,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;SACb,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;SACb,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;SAC5B,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC;SACvB,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;SAC5B,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;IAC/B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;SACb,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;SAC3B,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;SACb,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;SAC3B,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC;SACxB,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;AACjC,CAAC;AAED,2BAA2B,IAAI,EAAE,MAAM;IACrC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC9B,IAAM,CAAC,GAAG;QACR,IAAI,EAAE;YACJ,IAAI,EAAE,KAAK;YACX,OAAO,EAAE,OAAO;YAChB,MAAM,EAAE;gBACN,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,kBAAkB,EAAE;gBAC/D;oBACE,IAAI,EAAE,KAAK;oBACX,OAAO,EAAE,KAAK;oBACd,MAAM,EAAE;wBACN,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,GAAG,MAAM,CAAC,YAAY,EAAE,EAAE;wBAChF,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,GAAG,MAAM,CAAC,SAAS,EAAE,EAAE;wBAC1E,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,KAAK,EAAE,EAAE;wBAClE,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,WAAW,EAAE,EAAE;qBAC/E;iBACF;gBACD,EAAE,IAAI,EAAE,IAAI,EAAE;gBACd,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,aAAa,EAAE;gBAC1D,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,uBAAuB,EAAE;gBAC9C,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,2BAA2B,EAAE;gBAClD;oBACE,IAAI,EAAE,KAAK;oBACX,OAAO,EAAE,KAAK;oBACd,MAAM,EAAE;wBACN;4BACE,IAAI,EAAE,OAAO;4BACb,OAAO,EAAE,OAAO;4BAChB,MAAM,EAAE;gCACN;oCACE,IAAI,EAAE,OAAO;oCACb,MAAM,EAAE;wCACN;4CACE,IAAI,EAAE,IAAI;4CACV,MAAM,EAAE;gDACN,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,oBAAoB,EAAE;gDAC5D,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,EAAE;6CAC3D;yCACF;qCACF;iCACF;gCACD;oCACE,IAAI,EAAE,OAAO;oCACb,MAAM,EAAE;wCACN;4CACE,IAAI,EAAE,IAAI;4CACV,MAAM,EAAE;gDACN;oDACE,IAAI,EAAE,IAAI;oDACV,MAAM;wDACJ,IAAM,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC;wDAC3E,IAAM,MAAM,GAAG,KAAK,CAAC;wDACrB,MAAM,CAAC,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,iBAAiB,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;oDACnF,CAAC;iDACF;gDACD;oDACE,IAAI,EAAE,IAAI;oDACV,MAAM;wDACJ,IAAM,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC;wDACzE,IAAM,MAAM,GAAG,IAAI,CAAC;wDACpB,MAAM,CAAC,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;oDACjF,CAAC;iDACF;6CACF;yCACF;qCACF;iCACF;6BACF;yBACF;qBACF;iBACF;aACF;SACF;QACD,SAAS,EAAE;YACT,IAAI,EAAE,GAAG;YACT,MAAM,EAAE;gBACN,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,eAAe,EAAE;gBACtC,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,mBAAmB,EAAE;aAC3C;SACF;KACF,CAAC;IACF,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,uBAAuB,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe;IACjE,IAAI,UAAU,GAAG,kBAAkB,CAAC,CAAC,wBAAwB;IAC7D,EAAE,CAAC,CACD,CAAC,eAAe,IAAI,CAAC,UAAU,KAAK,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,eAAe,IAAI,CAAC,UAAU,KAAK,eAAe,CAAC,CAChH,CAAC,CAAC,CAAC;QAAC,UAAU,GAAG,kBAAkB,CAAC;IAAC,CAAC,CAAC,sBAAsB;IAC7D,EAAE,CAAC,CAAC,SAAS,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC;QACrC,EAAE,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YAClC,IAAM,UAAU,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACnD,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,iCAAiC,GAAG,UAAU,GAAG,YAAY,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;QAC1H,CAAC;IACH,CAAC;IACD,MAAM,CAAC,IAAI,CAAC;AACd,CAAC;AAED,qBAAqB,CAAC,EAAE,CAAC;IACvB,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,KAAK,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,KAAK,cAAc,CAAC,CAAC,CAAC,CAAC;QAC3E,MAAM,CAAC,OAAO,CAAC;IACjB,CAAC;IACD,MAAM,CAAC,KAAK,CAAC;AACf,CAAC;AAED,+BAA+B,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM;IAChD,IAAM,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;IAC9B,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QACrB,KAAK,cAAc;YAAE,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;YACf,KAAK,CAAC;QAC3B,KAAK,eAAe;YAAE,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;YACf,KAAK,CAAC;QAC5B,KAAK,cAAc;YAAE,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;YACf,KAAK,CAAC;QAC3B,KAAK,eAAe;YAAE,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;YACf,KAAK,CAAC;IAC9B,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,KAAK,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,KAAK,eAAe,CAAC,CAAC,CAAC,CAAC;QAC5E,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IAC/D,CAAC;IAAC,IAAI,CAAC,CAAC;QACN,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC;IACzF,CAAC;IACD,MAAM,CAAC,CAAC,CAAC;AACX,CAAC;AAED,+BAA+B,KAAK,EAAE,CAAC,EAAE,CAAC;IACxC,IAAM,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;IAC9B,IAAM,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IAC3B,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,KAAK,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,KAAK,eAAe,CAAC,CAAC,CAAC,CAAC;QAC5E,IAAM,QAAQ,GAAG,CAAC,GAAG,KAAK,CAAC;QAC3B,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC5B,CAAC;IAAC,IAAI,CAAC,CAAC;QACN,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC;IACD,MAAM,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;AACzD,CAAC"} \ No newline at end of file diff --git a/extensions/tabs/dist/graph.js.map b/extensions/tabs/dist/graph.js.map new file mode 100644 index 0000000..3d4ae2f --- /dev/null +++ b/extensions/tabs/dist/graph.js.map @@ -0,0 +1 @@ +{"version":3,"file":"graph.js","sourceRoot":"","sources":["../src/graph.ts"],"names":[],"mappings":"AAAA,4DAA4D;AAC5D,kCAAkC;AAElC,eAAe;AAEf,0CAA0C;AAC1C;IACE;QACE,IAAI,CAAC,MAAM,GAAa,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAA;QACpE,IAAI,CAAC,OAAO,GAAY,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAA;QACpE,IAAI,CAAC,UAAU,GAAS,GAAG,CAAA;QAC3B,IAAI,CAAC,WAAW,GAAQ,GAAG,CAAA;QAC3B,IAAI,CAAC,UAAU,GAAS,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAA;QAC9E,IAAI,CAAC,WAAW,GAAQ,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAA;QAC/E,IAAI,CAAC,KAAK,GAAc,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAA;QAChF,IAAI,CAAC,MAAM,GAAa,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;QACjF,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAA;IAC3B,CAAC;IAED,sBAAM,GAAN;QACE,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE;aACpB,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;aACnB,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;IAC5B,CAAC;IAED,sBAAM,GAAN;QACE,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE;aACpB,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC;aACtC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC,CAAA;IAChC,CAAC;IACH,YAAC;AAAD,CAAC,AAxBD,IAwBC"} \ No newline at end of file diff --git a/extensions/tabs/dist/init.js.map b/extensions/tabs/dist/init.js.map new file mode 100644 index 0000000..90f4679 --- /dev/null +++ b/extensions/tabs/dist/init.js.map @@ -0,0 +1 @@ +{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":"AAAA,uCAAuC;AAEvC,aAAa;AACb,8BAA8B;AAC9B,kCAAkC;AAClC,eAAe;AACf,4BAA4B;AAC5B,QAAQ;AACR,MAAM;AACN,KAAK;AAEL,oBAAoB;AACpB,wCAAwC;AACxC,qDAAqD;AACrD,6HAA6H;AAC7H,uJAAuJ;AACvJ,uBAAuB;AACvB,yBAAyB;AACzB,8CAA8C;AAC9C,gLAAgL;AAChL,sGAAsG;AACtG,4GAA4G;AAC5G,cAAc;AACd,aAAa;AAEb,2BAA2B;AAC3B,wEAAwE;AACxE,mBAAmB;AACnB,iDAAiD;AACjD,0FAA0F;AAC1F,4EAA4E;AAC5E,eAAe;AACf,YAAY;AAEZ,6BAA6B;AAC7B,0EAA0E;AAC1E,YAAY;AACZ,WAAW;AACX,SAAS;AACT,OAAO;AACP,KAAK;AAEL,kEAAkE;AAClE,IAAM,QAAQ,GAAG,eAAe,CAAC;AACjC,qBAAqB,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAC"} \ No newline at end of file diff --git a/extensions/tabs/dist/matrix.js.map b/extensions/tabs/dist/matrix.js.map new file mode 100644 index 0000000..9049f34 --- /dev/null +++ b/extensions/tabs/dist/matrix.js.map @@ -0,0 +1 @@ +{"version":3,"file":"matrix.js","sourceRoot":"","sources":["../src/matrix.ts"],"names":[],"mappings":"AAAA,4DAA4D;AAC5D,kCAAkC;AAElC,0CAA0C;AAC1C;IACE;QACE,IAAI,CAAC,CAAC,GAAI,CAAC,CAAC;QACZ,IAAI,CAAC,CAAC,GAAI,CAAC,CAAC;QACZ,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;QACZ,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;QACZ,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;QACZ,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;IACd,CAAC;IAEM,4BAAW,GAAlB;QACE,IAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;QAClF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,GAAG,GAAG,CAAA;IACzC,CAAC;IAEM,sBAAK,GAAZ;QACE,IAAM,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,CAAA;QAC1D,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,GAAG,CAAC,GAAG,GAAG,CAAA;IACnC,CAAC;IAEM,6BAAY,GAAnB;QACE,IAAM,SAAS,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,CAAA;QAC/C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC,GAAG,GAAG,CAAA;IAC1C,CAAC;IAEM,0BAAS,GAAhB;QACE,IAAM,MAAM,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,CAAA;QAC5C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,GAAG,GAAG,CAAA;IACvC,CAAC;IACH,aAAC;AAAD,CAAC,AA7BD,IA6BC"} \ No newline at end of file diff --git a/extensions/tabs/matrix.css b/extensions/tabs/matrix.css new file mode 100644 index 0000000..9f2edf4 --- /dev/null +++ b/extensions/tabs/matrix.css @@ -0,0 +1,81 @@ +@keyframes buttonTransition { + from { + r: 5; + } + + to { + r: 8; + } +} + +circle.active, +circle.clicked { + animation-duration: .5s; + animation-iteration-count: 1; + animation-name: buttonTransition; + animation-fill-mode: forwards; +} + +circle.clicked { + opacity: 1; +} + +div.tooltip { + position: absolute; + text-align: center; + padding: 5px; + font: 12px sans-serif; + background: lightsteelblue; + border: 0px; + border-radius: 8px; + pointer-events: none; +} + +#testData, +#slicers { + padding: 5px; +} + +#slicers { + background-color: lightgrey; + height: 100%; +} + +body { + padding: 10px; + font: 10px sans-serif; +} + +.axis line, +.axis path { + fill: none; + stroke: #000; + shape-rendering: crispEdges; +} + +.arrow { + stroke: #000; + stroke-width: 1.5px; +} + +.outer, +.inner { + shape-rendering: crispEdges; +} + +.outer { + fill: none; + stroke: #000; +} + +.inner { + fill: #ccc; + stroke: #000; + stroke-dasharray: 3, 4; +} + +.heading { + padding: 10px 0px 5px 0px; + font: 16px sans-serif; + font-weight: bold +} diff --git a/extensions/tabs/metadata.json b/extensions/tabs/metadata.json new file mode 100644 index 0000000..c8e069e --- /dev/null +++ b/extensions/tabs/metadata.json @@ -0,0 +1,2606 @@ +[ + { + "testName": "TruePositiveIntent('PlayMusic', 'start playing music')", + "group": "PlayMusic", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "start playing music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.92854017, + "entities": [], + "text": "start playing music", + "intent": "PlayMusic" + }, + "score": 0.92854017, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "TruePositiveIntent('PlayMusic', 'play music')", + "group": "PlayMusic", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "play music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.907853544, + "entities": [], + "text": "play music", + "intent": "PlayMusic" + }, + "score": 0.907853544, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "TruePositiveIntent('PlayMusic', 'listen to hip hop')", + "group": "PlayMusic", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "listen to hip hop", + "intent": "PlayMusic", + "entities": [ + { + "entityType": "Genre", + "matchText": "hip hop" + } + ] + }, + "actualUtterance": { + "score": 0.539262056, + "entities": [ + { + "score": 0.662749946, + "entityType": "Genre", + "matchText": "hip" + } + ], + "text": "listen to hip hop", + "intent": "PlayMusic" + }, + "score": 0.539262056, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "FalseNegativeIntent('Skip', 'skip this one')", + "group": "Skip", + "resultKind": "FalseNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "skip this one", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.449134439, + "entities": [], + "text": "skip this one", + "intent": "None" + }, + "score": 0.449134439, + "because": "Actual intent is 'None', expected 'Skip'", + "categories": [ + "Intent", + "FalseNegative" + ] + }, + { + "testName": "TruePositiveIntent('Skip', 'next please')", + "group": "Skip", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "next please", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.469399869, + "entities": [], + "text": "next please", + "intent": "Skip" + }, + "score": 0.469399869, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "FalseNegativeIntent('Skip', 'go forward')", + "group": "Skip", + "resultKind": "FalseNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "go forward", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.463593155, + "entities": [], + "text": "go forward", + "intent": "None" + }, + "score": 0.463593155, + "because": "Actual intent is 'None', expected 'Skip'", + "categories": [ + "Intent", + "FalseNegative" + ] + }, + { + "testName": "TrueNegativeIntent('is it cold out')", + "resultKind": "TrueNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "is it cold out", + "intent": "None" + }, + "actualUtterance": { + "score": 0.873349249, + "entities": [], + "text": "is it cold out", + "intent": "None" + }, + "score": 0.873349249, + "because": "Both intents are 'None'.", + "categories": [ + "Intent", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeIntent('how many days until Christmas')", + "resultKind": "TrueNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "how many days until Christmas", + "intent": "None" + }, + "actualUtterance": { + "score": 0.42890653, + "entities": [], + "text": "how many days until Christmas", + "intent": "None" + }, + "score": 0.42890653, + "because": "Both intents are 'None'.", + "categories": [ + "Intent", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeIntent('what's the weather like')", + "resultKind": "TrueNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "what's the weather like", + "intent": "None" + }, + "actualUtterance": { + "score": 0.5407783, + "entities": [], + "text": "what's the weather like", + "intent": "None" + }, + "score": 0.5407783, + "because": "Both intents are 'None'.", + "categories": [ + "Intent", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('start playing music')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "start playing music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.92854017, + "entities": [], + "text": "start playing music", + "intent": "PlayMusic" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('play music')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "play music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.907853544, + "entities": [], + "text": "play music", + "intent": "PlayMusic" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "FalseNegativeEntity('Genre', '\"hip hop\"', 'listen to hip hop')", + "group": "Genre", + "resultKind": "FalseNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "listen to hip hop", + "intent": "PlayMusic", + "entities": [ + { + "entityType": "Genre", + "matchText": "hip hop" + } + ] + }, + "actualUtterance": { + "score": 0.539262056, + "entities": [ + { + "score": 0.662749946, + "entityType": "Genre", + "matchText": "hip" + } + ], + "text": "listen to hip hop", + "intent": "PlayMusic" + }, + "because": "Actual utterance does not have entity matching 'hip hop'.", + "categories": [ + "Entity", + "FalseNegative" + ] + }, + { + "testName": "FalsePositiveEntity('Genre', '\"hip\"', 'listen to hip hop')", + "group": "Genre", + "resultKind": "FalsePositive", + "targetKind": "Entity", + "expectedUtterance": { + "text": "listen to hip hop", + "intent": "PlayMusic", + "entities": [ + { + "entityType": "Genre", + "matchText": "hip hop" + } + ] + }, + "actualUtterance": { + "score": 0.539262056, + "entities": [ + { + "score": 0.662749946, + "entityType": "Genre", + "matchText": "hip" + } + ], + "text": "listen to hip hop", + "intent": "PlayMusic" + }, + "score": 0.662749946, + "because": "Expected utterance does not have entity matching 'hip'.", + "categories": [ + "Entity", + "FalsePositive" + ] + }, + { + "testName": "TrueNegativeEntity('skip this one')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "skip this one", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.449134439, + "entities": [], + "text": "skip this one", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('next please')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "next please", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.469399869, + "entities": [], + "text": "next please", + "intent": "Skip" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('go forward')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "go forward", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.463593155, + "entities": [], + "text": "go forward", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('is it cold out')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "is it cold out", + "intent": "None" + }, + "actualUtterance": { + "score": 0.873349249, + "entities": [], + "text": "is it cold out", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('how many days until Christmas')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "how many days until Christmas", + "intent": "None" + }, + "actualUtterance": { + "score": 0.42890653, + "entities": [], + "text": "how many days until Christmas", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('what's the weather like')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "what's the weather like", + "intent": "None" + }, + "actualUtterance": { + "score": 0.5407783, + "entities": [], + "text": "what's the weather like", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TruePositiveIntent('PlayMusic', 'start playing music')", + "group": "PlayMusic", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "start playing music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.92854017, + "entities": [], + "text": "start playing music", + "intent": "PlayMusic" + }, + "score": 0.92854017, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "TruePositiveIntent('PlayMusic', 'play music')", + "group": "PlayMusic", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "play music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.907853544, + "entities": [], + "text": "play music", + "intent": "PlayMusic" + }, + "score": 0.907853544, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "TruePositiveIntent('PlayMusic', 'listen to hip hop')", + "group": "PlayMusic", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "listen to hip hop", + "intent": "PlayMusic", + "entities": [ + { + "entityType": "Genre", + "matchText": "hip hop" + } + ] + }, + "actualUtterance": { + "score": 0.539262056, + "entities": [ + { + "score": 0.662749946, + "entityType": "Genre", + "matchText": "hip" + } + ], + "text": "listen to hip hop", + "intent": "PlayMusic" + }, + "score": 0.539262056, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "FalseNegativeIntent('Skip', 'skip this one')", + "group": "Skip", + "resultKind": "FalseNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "skip this one", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.449134439, + "entities": [], + "text": "skip this one", + "intent": "None" + }, + "score": 0.449134439, + "because": "Actual intent is 'None', expected 'Skip'", + "categories": [ + "Intent", + "FalseNegative" + ] + }, + { + "testName": "TruePositiveIntent('Skip', 'next please')", + "group": "Skip", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "next please", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.469399869, + "entities": [], + "text": "next please", + "intent": "Skip" + }, + "score": 0.469399869, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "FalseNegativeIntent('Skip', 'go forward')", + "group": "Skip", + "resultKind": "FalseNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "go forward", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.463593155, + "entities": [], + "text": "go forward", + "intent": "None" + }, + "score": 0.463593155, + "because": "Actual intent is 'None', expected 'Skip'", + "categories": [ + "Intent", + "FalseNegative" + ] + }, + { + "testName": "TrueNegativeIntent('is it cold out')", + "resultKind": "TrueNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "is it cold out", + "intent": "None" + }, + "actualUtterance": { + "score": 0.873349249, + "entities": [], + "text": "is it cold out", + "intent": "None" + }, + "score": 0.873349249, + "because": "Both intents are 'None'.", + "categories": [ + "Intent", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeIntent('how many days until Christmas')", + "resultKind": "TrueNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "how many days until Christmas", + "intent": "None" + }, + "actualUtterance": { + "score": 0.42890653, + "entities": [], + "text": "how many days until Christmas", + "intent": "None" + }, + "score": 0.42890653, + "because": "Both intents are 'None'.", + "categories": [ + "Intent", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeIntent('what's the weather like')", + "resultKind": "TrueNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "what's the weather like", + "intent": "None" + }, + "actualUtterance": { + "score": 0.5407783, + "entities": [], + "text": "what's the weather like", + "intent": "None" + }, + "score": 0.5407783, + "because": "Both intents are 'None'.", + "categories": [ + "Intent", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('start playing music')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "start playing music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.92854017, + "entities": [], + "text": "start playing music", + "intent": "PlayMusic" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('play music')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "play music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.907853544, + "entities": [], + "text": "play music", + "intent": "PlayMusic" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "FalseNegativeEntity('Genre', '\"hip hop\"', 'listen to hip hop')", + "group": "Genre", + "resultKind": "FalseNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "listen to hip hop", + "intent": "PlayMusic", + "entities": [ + { + "entityType": "Genre", + "matchText": "hip hop" + } + ] + }, + "actualUtterance": { + "score": 0.539262056, + "entities": [ + { + "score": 0.662749946, + "entityType": "Genre", + "matchText": "hip" + } + ], + "text": "listen to hip hop", + "intent": "PlayMusic" + }, + "because": "Actual utterance does not have entity matching 'hip hop'.", + "categories": [ + "Entity", + "FalseNegative" + ] + }, + { + "testName": "FalsePositiveEntity('Genre', '\"hip\"', 'listen to hip hop')", + "group": "Genre", + "resultKind": "FalsePositive", + "targetKind": "Entity", + "expectedUtterance": { + "text": "listen to hip hop", + "intent": "PlayMusic", + "entities": [ + { + "entityType": "Genre", + "matchText": "hip hop" + } + ] + }, + "actualUtterance": { + "score": 0.539262056, + "entities": [ + { + "score": 0.662749946, + "entityType": "Genre", + "matchText": "hip" + } + ], + "text": "listen to hip hop", + "intent": "PlayMusic" + }, + "score": 0.662749946, + "because": "Expected utterance does not have entity matching 'hip'.", + "categories": [ + "Entity", + "FalsePositive" + ] + }, + { + "testName": "TrueNegativeEntity('skip this one')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "skip this one", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.449134439, + "entities": [], + "text": "skip this one", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('next please')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "next please", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.469399869, + "entities": [], + "text": "next please", + "intent": "Skip" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('go forward')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "go forward", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.463593155, + "entities": [], + "text": "go forward", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('is it cold out')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "is it cold out", + "intent": "None" + }, + "actualUtterance": { + "score": 0.873349249, + "entities": [], + "text": "is it cold out", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('how many days until Christmas')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "how many days until Christmas", + "intent": "None" + }, + "actualUtterance": { + "score": 0.42890653, + "entities": [], + "text": "how many days until Christmas", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('what's the weather like')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "what's the weather like", + "intent": "None" + }, + "actualUtterance": { + "score": 0.5407783, + "entities": [], + "text": "what's the weather like", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TruePositiveIntent('PlayMusic', 'start playing music')", + "group": "PlayMusic", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "start playing music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.92854017, + "entities": [], + "text": "start playing music", + "intent": "PlayMusic" + }, + "score": 0.92854017, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "TruePositiveIntent('PlayMusic', 'play music')", + "group": "PlayMusic", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "play music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.907853544, + "entities": [], + "text": "play music", + "intent": "PlayMusic" + }, + "score": 0.907853544, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "TruePositiveIntent('PlayMusic', 'listen to hip hop')", + "group": "PlayMusic", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "listen to hip hop", + "intent": "PlayMusic", + "entities": [ + { + "entityType": "Genre", + "matchText": "hip hop" + } + ] + }, + "actualUtterance": { + "score": 0.539262056, + "entities": [ + { + "score": 0.662749946, + "entityType": "Genre", + "matchText": "hip" + } + ], + "text": "listen to hip hop", + "intent": "PlayMusic" + }, + "score": 0.539262056, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "FalseNegativeIntent('Skip', 'skip this one')", + "group": "Skip", + "resultKind": "FalseNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "skip this one", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.449134439, + "entities": [], + "text": "skip this one", + "intent": "None" + }, + "score": 0.449134439, + "because": "Actual intent is 'None', expected 'Skip'", + "categories": [ + "Intent", + "FalseNegative" + ] + }, + { + "testName": "TruePositiveIntent('Skip', 'next please')", + "group": "Skip", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "next please", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.469399869, + "entities": [], + "text": "next please", + "intent": "Skip" + }, + "score": 0.469399869, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "FalseNegativeIntent('Skip', 'go forward')", + "group": "Skip", + "resultKind": "FalseNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "go forward", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.463593155, + "entities": [], + "text": "go forward", + "intent": "None" + }, + "score": 0.463593155, + "because": "Actual intent is 'None', expected 'Skip'", + "categories": [ + "Intent", + "FalseNegative" + ] + }, + { + "testName": "TrueNegativeIntent('is it cold out')", + "resultKind": "TrueNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "is it cold out", + "intent": "None" + }, + "actualUtterance": { + "score": 0.873349249, + "entities": [], + "text": "is it cold out", + "intent": "None" + }, + "score": 0.873349249, + "because": "Both intents are 'None'.", + "categories": [ + "Intent", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeIntent('how many days until Christmas')", + "resultKind": "TrueNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "how many days until Christmas", + "intent": "None" + }, + "actualUtterance": { + "score": 0.42890653, + "entities": [], + "text": "how many days until Christmas", + "intent": "None" + }, + "score": 0.42890653, + "because": "Both intents are 'None'.", + "categories": [ + "Intent", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeIntent('what's the weather like')", + "resultKind": "TrueNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "what's the weather like", + "intent": "None" + }, + "actualUtterance": { + "score": 0.5407783, + "entities": [], + "text": "what's the weather like", + "intent": "None" + }, + "score": 0.5407783, + "because": "Both intents are 'None'.", + "categories": [ + "Intent", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('start playing music')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "start playing music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.92854017, + "entities": [], + "text": "start playing music", + "intent": "PlayMusic" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('play music')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "play music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.907853544, + "entities": [], + "text": "play music", + "intent": "PlayMusic" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "FalseNegativeEntity('Genre', '\"hip hop\"', 'listen to hip hop')", + "group": "Genre", + "resultKind": "FalseNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "listen to hip hop", + "intent": "PlayMusic", + "entities": [ + { + "entityType": "Genre", + "matchText": "hip hop" + } + ] + }, + "actualUtterance": { + "score": 0.539262056, + "entities": [ + { + "score": 0.662749946, + "entityType": "Genre", + "matchText": "hip" + } + ], + "text": "listen to hip hop", + "intent": "PlayMusic" + }, + "because": "Actual utterance does not have entity matching 'hip hop'.", + "categories": [ + "Entity", + "FalseNegative" + ] + }, + { + "testName": "FalsePositiveEntity('Genre', '\"hip\"', 'listen to hip hop')", + "group": "Genre", + "resultKind": "FalsePositive", + "targetKind": "Entity", + "expectedUtterance": { + "text": "listen to hip hop", + "intent": "PlayMusic", + "entities": [ + { + "entityType": "Genre", + "matchText": "hip hop" + } + ] + }, + "actualUtterance": { + "score": 0.539262056, + "entities": [ + { + "score": 0.662749946, + "entityType": "Genre", + "matchText": "hip" + } + ], + "text": "listen to hip hop", + "intent": "PlayMusic" + }, + "score": 0.662749946, + "because": "Expected utterance does not have entity matching 'hip'.", + "categories": [ + "Entity", + "FalsePositive" + ] + }, + { + "testName": "TrueNegativeEntity('skip this one')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "skip this one", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.449134439, + "entities": [], + "text": "skip this one", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('next please')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "next please", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.469399869, + "entities": [], + "text": "next please", + "intent": "Skip" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('go forward')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "go forward", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.463593155, + "entities": [], + "text": "go forward", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('is it cold out')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "is it cold out", + "intent": "None" + }, + "actualUtterance": { + "score": 0.873349249, + "entities": [], + "text": "is it cold out", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('how many days until Christmas')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "how many days until Christmas", + "intent": "None" + }, + "actualUtterance": { + "score": 0.42890653, + "entities": [], + "text": "how many days until Christmas", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('what's the weather like')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "what's the weather like", + "intent": "None" + }, + "actualUtterance": { + "score": 0.5407783, + "entities": [], + "text": "what's the weather like", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TruePositiveIntent('PlayMusic', 'start playing music')", + "group": "PlayMusic", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "start playing music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.92854017, + "entities": [], + "text": "start playing music", + "intent": "PlayMusic" + }, + "score": 0.92854017, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "TruePositiveIntent('PlayMusic', 'play music')", + "group": "PlayMusic", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "play music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.907853544, + "entities": [], + "text": "play music", + "intent": "PlayMusic" + }, + "score": 0.907853544, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "TruePositiveIntent('PlayMusic', 'listen to hip hop')", + "group": "PlayMusic", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "listen to hip hop", + "intent": "PlayMusic", + "entities": [ + { + "entityType": "Genre", + "matchText": "hip hop" + } + ] + }, + "actualUtterance": { + "score": 0.539262056, + "entities": [ + { + "score": 0.662749946, + "entityType": "Genre", + "matchText": "hip" + } + ], + "text": "listen to hip hop", + "intent": "PlayMusic" + }, + "score": 0.539262056, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "FalseNegativeIntent('Skip', 'skip this one')", + "group": "Skip", + "resultKind": "FalseNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "skip this one", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.449134439, + "entities": [], + "text": "skip this one", + "intent": "None" + }, + "score": 0.449134439, + "because": "Actual intent is 'None', expected 'Skip'", + "categories": [ + "Intent", + "FalseNegative" + ] + }, + { + "testName": "TruePositiveIntent('Skip', 'next please')", + "group": "Skip", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "next please", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.469399869, + "entities": [], + "text": "next please", + "intent": "Skip" + }, + "score": 0.469399869, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "FalseNegativeIntent('Skip', 'go forward')", + "group": "Skip", + "resultKind": "FalseNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "go forward", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.463593155, + "entities": [], + "text": "go forward", + "intent": "None" + }, + "score": 0.463593155, + "because": "Actual intent is 'None', expected 'Skip'", + "categories": [ + "Intent", + "FalseNegative" + ] + }, + { + "testName": "TrueNegativeIntent('is it cold out')", + "resultKind": "TrueNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "is it cold out", + "intent": "None" + }, + "actualUtterance": { + "score": 0.873349249, + "entities": [], + "text": "is it cold out", + "intent": "None" + }, + "score": 0.873349249, + "because": "Both intents are 'None'.", + "categories": [ + "Intent", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeIntent('how many days until Christmas')", + "resultKind": "TrueNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "how many days until Christmas", + "intent": "None" + }, + "actualUtterance": { + "score": 0.42890653, + "entities": [], + "text": "how many days until Christmas", + "intent": "None" + }, + "score": 0.42890653, + "because": "Both intents are 'None'.", + "categories": [ + "Intent", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeIntent('what's the weather like')", + "resultKind": "TrueNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "what's the weather like", + "intent": "None" + }, + "actualUtterance": { + "score": 0.5407783, + "entities": [], + "text": "what's the weather like", + "intent": "None" + }, + "score": 0.5407783, + "because": "Both intents are 'None'.", + "categories": [ + "Intent", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('start playing music')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "start playing music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.92854017, + "entities": [], + "text": "start playing music", + "intent": "PlayMusic" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('play music')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "play music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.907853544, + "entities": [], + "text": "play music", + "intent": "PlayMusic" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "FalseNegativeEntity('Genre', '\"hip hop\"', 'listen to hip hop')", + "group": "Genre", + "resultKind": "FalseNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "listen to hip hop", + "intent": "PlayMusic", + "entities": [ + { + "entityType": "Genre", + "matchText": "hip hop" + } + ] + }, + "actualUtterance": { + "score": 0.539262056, + "entities": [ + { + "score": 0.662749946, + "entityType": "Genre", + "matchText": "hip" + } + ], + "text": "listen to hip hop", + "intent": "PlayMusic" + }, + "because": "Actual utterance does not have entity matching 'hip hop'.", + "categories": [ + "Entity", + "FalseNegative" + ] + }, + { + "testName": "FalsePositiveEntity('Genre', '\"hip\"', 'listen to hip hop')", + "group": "Genre", + "resultKind": "FalsePositive", + "targetKind": "Entity", + "expectedUtterance": { + "text": "listen to hip hop", + "intent": "PlayMusic", + "entities": [ + { + "entityType": "Genre", + "matchText": "hip hop" + } + ] + }, + "actualUtterance": { + "score": 0.539262056, + "entities": [ + { + "score": 0.662749946, + "entityType": "Genre", + "matchText": "hip" + } + ], + "text": "listen to hip hop", + "intent": "PlayMusic" + }, + "score": 0.662749946, + "because": "Expected utterance does not have entity matching 'hip'.", + "categories": [ + "Entity", + "FalsePositive" + ] + }, + { + "testName": "TrueNegativeEntity('skip this one')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "skip this one", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.449134439, + "entities": [], + "text": "skip this one", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('next please')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "next please", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.469399869, + "entities": [], + "text": "next please", + "intent": "Skip" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('go forward')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "go forward", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.463593155, + "entities": [], + "text": "go forward", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('is it cold out')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "is it cold out", + "intent": "None" + }, + "actualUtterance": { + "score": 0.873349249, + "entities": [], + "text": "is it cold out", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('how many days until Christmas')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "how many days until Christmas", + "intent": "None" + }, + "actualUtterance": { + "score": 0.42890653, + "entities": [], + "text": "how many days until Christmas", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('what's the weather like')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "what's the weather like", + "intent": "None" + }, + "actualUtterance": { + "score": 0.5407783, + "entities": [], + "text": "what's the weather like", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TruePositiveIntent('PlayMusic', 'start playing music')", + "group": "PlayMusic", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "start playing music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.92854017, + "entities": [], + "text": "start playing music", + "intent": "PlayMusic" + }, + "score": 0.92854017, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "TruePositiveIntent('PlayMusic', 'play music')", + "group": "PlayMusic", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "play music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.907853544, + "entities": [], + "text": "play music", + "intent": "PlayMusic" + }, + "score": 0.907853544, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "TruePositiveIntent('PlayMusic', 'listen to hip hop')", + "group": "PlayMusic", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "listen to hip hop", + "intent": "PlayMusic", + "entities": [ + { + "entityType": "Genre", + "matchText": "hip hop" + } + ] + }, + "actualUtterance": { + "score": 0.539262056, + "entities": [ + { + "score": 0.662749946, + "entityType": "Genre", + "matchText": "hip" + } + ], + "text": "listen to hip hop", + "intent": "PlayMusic" + }, + "score": 0.539262056, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "FalseNegativeIntent('Skip', 'skip this one')", + "group": "Skip", + "resultKind": "FalseNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "skip this one", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.449134439, + "entities": [], + "text": "skip this one", + "intent": "None" + }, + "score": 0.449134439, + "because": "Actual intent is 'None', expected 'Skip'", + "categories": [ + "Intent", + "FalseNegative" + ] + }, + { + "testName": "TruePositiveIntent('Skip', 'next please')", + "group": "Skip", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "next please", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.469399869, + "entities": [], + "text": "next please", + "intent": "Skip" + }, + "score": 0.469399869, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "FalseNegativeIntent('Skip', 'go forward')", + "group": "Skip", + "resultKind": "FalseNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "go forward", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.463593155, + "entities": [], + "text": "go forward", + "intent": "None" + }, + "score": 0.463593155, + "because": "Actual intent is 'None', expected 'Skip'", + "categories": [ + "Intent", + "FalseNegative" + ] + }, + { + "testName": "TrueNegativeIntent('is it cold out')", + "resultKind": "TrueNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "is it cold out", + "intent": "None" + }, + "actualUtterance": { + "score": 0.873349249, + "entities": [], + "text": "is it cold out", + "intent": "None" + }, + "score": 0.873349249, + "because": "Both intents are 'None'.", + "categories": [ + "Intent", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeIntent('how many days until Christmas')", + "resultKind": "TrueNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "how many days until Christmas", + "intent": "None" + }, + "actualUtterance": { + "score": 0.42890653, + "entities": [], + "text": "how many days until Christmas", + "intent": "None" + }, + "score": 0.42890653, + "because": "Both intents are 'None'.", + "categories": [ + "Intent", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeIntent('what's the weather like')", + "resultKind": "TrueNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "what's the weather like", + "intent": "None" + }, + "actualUtterance": { + "score": 0.5407783, + "entities": [], + "text": "what's the weather like", + "intent": "None" + }, + "score": 0.5407783, + "because": "Both intents are 'None'.", + "categories": [ + "Intent", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('start playing music')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "start playing music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.92854017, + "entities": [], + "text": "start playing music", + "intent": "PlayMusic" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('play music')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "play music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.907853544, + "entities": [], + "text": "play music", + "intent": "PlayMusic" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "FalseNegativeEntity('Genre', '\"hip hop\"', 'listen to hip hop')", + "group": "Genre", + "resultKind": "FalseNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "listen to hip hop", + "intent": "PlayMusic", + "entities": [ + { + "entityType": "Genre", + "matchText": "hip hop" + } + ] + }, + "actualUtterance": { + "score": 0.539262056, + "entities": [ + { + "score": 0.662749946, + "entityType": "Genre", + "matchText": "hip" + } + ], + "text": "listen to hip hop", + "intent": "PlayMusic" + }, + "because": "Actual utterance does not have entity matching 'hip hop'.", + "categories": [ + "Entity", + "FalseNegative" + ] + }, + { + "testName": "FalsePositiveEntity('Genre', '\"hip\"', 'listen to hip hop')", + "group": "Genre", + "resultKind": "FalsePositive", + "targetKind": "Entity", + "expectedUtterance": { + "text": "listen to hip hop", + "intent": "PlayMusic", + "entities": [ + { + "entityType": "Genre", + "matchText": "hip hop" + } + ] + }, + "actualUtterance": { + "score": 0.539262056, + "entities": [ + { + "score": 0.662749946, + "entityType": "Genre", + "matchText": "hip" + } + ], + "text": "listen to hip hop", + "intent": "PlayMusic" + }, + "score": 0.662749946, + "because": "Expected utterance does not have entity matching 'hip'.", + "categories": [ + "Entity", + "FalsePositive" + ] + }, + { + "testName": "TrueNegativeEntity('skip this one')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "skip this one", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.449134439, + "entities": [], + "text": "skip this one", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('next please')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "next please", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.469399869, + "entities": [], + "text": "next please", + "intent": "Skip" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('go forward')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "go forward", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.463593155, + "entities": [], + "text": "go forward", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('is it cold out')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "is it cold out", + "intent": "None" + }, + "actualUtterance": { + "score": 0.873349249, + "entities": [], + "text": "is it cold out", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('how many days until Christmas')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "how many days until Christmas", + "intent": "None" + }, + "actualUtterance": { + "score": 0.42890653, + "entities": [], + "text": "how many days until Christmas", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('what's the weather like')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "what's the weather like", + "intent": "None" + }, + "actualUtterance": { + "score": 0.5407783, + "entities": [], + "text": "what's the weather like", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TruePositiveIntent('PlayMusic', 'start playing music')", + "group": "PlayMusic", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "start playing music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.92854017, + "entities": [], + "text": "start playing music", + "intent": "PlayMusic" + }, + "score": 0.92854017, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "TruePositiveIntent('PlayMusic', 'play music')", + "group": "PlayMusic", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "play music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.907853544, + "entities": [], + "text": "play music", + "intent": "PlayMusic" + }, + "score": 0.907853544, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "TruePositiveIntent('PlayMusic', 'listen to hip hop')", + "group": "PlayMusic", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "listen to hip hop", + "intent": "PlayMusic", + "entities": [ + { + "entityType": "Genre", + "matchText": "hip hop" + } + ] + }, + "actualUtterance": { + "score": 0.539262056, + "entities": [ + { + "score": 0.662749946, + "entityType": "Genre", + "matchText": "hip" + } + ], + "text": "listen to hip hop", + "intent": "PlayMusic" + }, + "score": 0.539262056, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "FalseNegativeIntent('Skip', 'skip this one')", + "group": "Skip", + "resultKind": "FalseNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "skip this one", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.449134439, + "entities": [], + "text": "skip this one", + "intent": "None" + }, + "score": 0.449134439, + "because": "Actual intent is 'None', expected 'Skip'", + "categories": [ + "Intent", + "FalseNegative" + ] + }, + { + "testName": "TruePositiveIntent('Skip', 'next please')", + "group": "Skip", + "resultKind": "TruePositive", + "targetKind": "Intent", + "expectedUtterance": { + "text": "next please", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.469399869, + "entities": [], + "text": "next please", + "intent": "Skip" + }, + "score": 0.469399869, + "because": "Utterances have matching intent.", + "categories": [ + "Intent", + "TruePositive" + ] + }, + { + "testName": "FalseNegativeIntent('Skip', 'go forward')", + "group": "Skip", + "resultKind": "FalseNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "go forward", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.463593155, + "entities": [], + "text": "go forward", + "intent": "None" + }, + "score": 0.463593155, + "because": "Actual intent is 'None', expected 'Skip'", + "categories": [ + "Intent", + "FalseNegative" + ] + }, + { + "testName": "TrueNegativeIntent('is it cold out')", + "resultKind": "TrueNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "is it cold out", + "intent": "None" + }, + "actualUtterance": { + "score": 0.873349249, + "entities": [], + "text": "is it cold out", + "intent": "None" + }, + "score": 0.873349249, + "because": "Both intents are 'None'.", + "categories": [ + "Intent", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeIntent('how many days until Christmas')", + "resultKind": "TrueNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "how many days until Christmas", + "intent": "None" + }, + "actualUtterance": { + "score": 0.42890653, + "entities": [], + "text": "how many days until Christmas", + "intent": "None" + }, + "score": 0.42890653, + "because": "Both intents are 'None'.", + "categories": [ + "Intent", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeIntent('what's the weather like')", + "resultKind": "TrueNegative", + "targetKind": "Intent", + "expectedUtterance": { + "text": "what's the weather like", + "intent": "None" + }, + "actualUtterance": { + "score": 0.5407783, + "entities": [], + "text": "what's the weather like", + "intent": "None" + }, + "score": 0.5407783, + "because": "Both intents are 'None'.", + "categories": [ + "Intent", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('start playing music')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "start playing music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.92854017, + "entities": [], + "text": "start playing music", + "intent": "PlayMusic" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('play music')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "play music", + "intent": "PlayMusic" + }, + "actualUtterance": { + "score": 0.907853544, + "entities": [], + "text": "play music", + "intent": "PlayMusic" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "FalseNegativeEntity('Genre', '\"hip hop\"', 'listen to hip hop')", + "group": "Genre", + "resultKind": "FalseNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "listen to hip hop", + "intent": "PlayMusic", + "entities": [ + { + "entityType": "Genre", + "matchText": "hip hop" + } + ] + }, + "actualUtterance": { + "score": 0.539262056, + "entities": [ + { + "score": 0.662749946, + "entityType": "Genre", + "matchText": "hip" + } + ], + "text": "listen to hip hop", + "intent": "PlayMusic" + }, + "because": "Actual utterance does not have entity matching 'hip hop'.", + "categories": [ + "Entity", + "FalseNegative" + ] + }, + { + "testName": "FalsePositiveEntity('Genre', '\"hip\"', 'listen to hip hop')", + "group": "Genre", + "resultKind": "FalsePositive", + "targetKind": "Entity", + "expectedUtterance": { + "text": "listen to hip hop", + "intent": "PlayMusic", + "entities": [ + { + "entityType": "Genre", + "matchText": "hip hop" + } + ] + }, + "actualUtterance": { + "score": 0.539262056, + "entities": [ + { + "score": 0.662749946, + "entityType": "Genre", + "matchText": "hip" + } + ], + "text": "listen to hip hop", + "intent": "PlayMusic" + }, + "score": 0.662749946, + "because": "Expected utterance does not have entity matching 'hip'.", + "categories": [ + "Entity", + "FalsePositive" + ] + }, + { + "testName": "TrueNegativeEntity('skip this one')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "skip this one", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.449134439, + "entities": [], + "text": "skip this one", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('next please')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "next please", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.469399869, + "entities": [], + "text": "next please", + "intent": "Skip" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('go forward')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "go forward", + "intent": "Skip" + }, + "actualUtterance": { + "score": 0.463593155, + "entities": [], + "text": "go forward", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('is it cold out')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "is it cold out", + "intent": "None" + }, + "actualUtterance": { + "score": 0.873349249, + "entities": [], + "text": "is it cold out", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('how many days until Christmas')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "how many days until Christmas", + "intent": "None" + }, + "actualUtterance": { + "score": 0.42890653, + "entities": [], + "text": "how many days until Christmas", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + }, + { + "testName": "TrueNegativeEntity('what's the weather like')", + "resultKind": "TrueNegative", + "targetKind": "Entity", + "expectedUtterance": { + "text": "what's the weather like", + "intent": "None" + }, + "actualUtterance": { + "score": 0.5407783, + "entities": [], + "text": "what's the weather like", + "intent": "None" + }, + "because": "Neither utterances have entities.", + "categories": [ + "Entity", + "TrueNegative" + ] + } +] diff --git a/extensions/tabs/package.json b/extensions/tabs/package.json new file mode 100644 index 0000000..1b0cf5e --- /dev/null +++ b/extensions/tabs/package.json @@ -0,0 +1,9 @@ +{ + "dependencies": { + "@types/d3": "^5.7.2", + "awesome-typescript-loader": "^5.2.1", + "live-server": "^1.2.1", + "source-map-loader": "^0.2.4", + "vss-web-extension-sdk": "^5.141.0" + } +} diff --git a/extensions/tabs/src/confusion.ts b/extensions/tabs/src/confusion.ts new file mode 100644 index 0000000..e17a921 --- /dev/null +++ b/extensions/tabs/src/confusion.ts @@ -0,0 +1,350 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// d3 and json2html included in HTML script reference +/* global d3, json2html, Matrix, Graph */ + +// json2html uses curly brace syntax for templating + +function renderConfusionMatrix(filename, headers) { + let quadrantCountMax = 0; + let nestData, facetActualIntents, facetExpectedIntents; + let facetActualEntity, facetExpectedEntity; + + // Group elements into a hierarchial tree structure + // Explanation here: https://github.com/d3/d3-collection#nests + d3.json(filename, { headers }).then(function(data) { + nestData = d3.nest() + .key(function(d) { return d.resultKind }) + .entries(data); + facetExpectedIntents = d3.nest() + .key(function(d) { return d.expectedUtterance.intent }) + .entries(data); + facetActualIntents = d3.nest() + .key(function(d) { return d.actualUtterance.intent }) + .entries(data); + + facetActualEntity = d3.nest() + .key(function(d) { + let entityType = d.actualUtterance.entities.entityType; + let entities = d.actualUtterance.entities; + + // Loop through entities where an entityType is present + if (entityType == undefined) { + for (let index = 0; index < entities.length; index++) { + const element = entities[index]; + + return element.entityType; + }; + } else { + return; // TODO: Need to return something stating that there is no entity + } + }) + .entries(data); + + facetExpectedEntity = d3.nest() + .key(function(d) { + let entities = d.expectedUtterance.entities; + + if (entities != undefined) { + // Loop through entities where an entityType is present + for (let index = 0; index < entities.length; index++) { + const element = entities[index]; + + return element.entityType; + }; + } else { + return; // TODO: Need to return something stating that there is no entity + } + }) + .entries(data); + + for (let key = 0; key < nestData.length; key++) { + console.log(nestData[key]); + quadrantCountMax = Math.max(quadrantCountMax, nestData[key].values.length); + } + + const matrix = new Matrix(); + const graph = new Graph(); + graph.quadrantCountMax = quadrantCountMax; + + addGraph(graph); + addDataPoints(nestData, matrix, graph); + addFacets("Actual Intents" , facetActualIntents); + addFacets("Expected Intents", facetExpectedIntents); + + addFacets("Actual Entity" , facetActualEntity); + addFacets("Expected Entity", facetExpectedEntity); + }); +} + +function addFacets(section, data) { + let slicerDiv = d3.select("#facets").append("div"); + + slicerDiv.append("div") + .attr("class", "heading") + .text(section); + + slicerDiv.selectAll("p") + .data(data) + .enter() + .append("div") + .text(function(d) { return d.key; }); +} + +function addDataPoints(data, matrix, graph) { + for (let index = 0; index < data.length; index++) { + let tooltip = d3.select("body").append("div") + .attr("class", "tooltip") + .style("opacity", 0); + d3.select("svg") + .selectAll("elements") + .data(data[index].values) + .enter() + .append("circle") + .attr("r", 5) + .attr("cx", function(d, i) { return getXConfusionQuadrant(graph, d, i, matrix); }) + .attr("cy", function(d, i) { return getYConfusionQuadrant(graph, d, i); }) + .attr("stroke", function(d, i) { return getDotColor(d, i); }) + .attr("fill", function(d, i) { return getDotColor(d, i); }) + .attr("opacity", 0.5) + .on("mouseover", function(d, i) { + d3.selectAll(".active").classed("active", false); + d3.select("svg") + .append("circle") + .attr("cx", getXConfusionQuadrant(graph, d, i, matrix)) + .attr("cy", getYConfusionQuadrant(graph, d, i)) + .attr("stroke", getDotColor(d, i)) + .attr("fill", getDotColor(d, i)) + .attr("opacity", 0.5) + .classed("active", true) + .on("click", function() { + d3.selectAll(".clicked").classed("clicked", false); + const div = d3.select("#testData"); + div.html(getTestResultHtml(d, matrix)); + d3.selectAll(".active").classed("clicked", true); + }) + .on("mouseover", function() { + tooltip.transition() + .duration(500) + .style("opacity", 0.9); + tooltip.html(d.score || "not scored") + .style("left", (d3.event.pageX) + "px") + .style("top", (d3.event.pageY - 28) + "px"); + }) + .on("mouseout", function() { + tooltip.transition() + .duration(500) + .style("opacity", 0); + d3.selectAll(".active").classed("active", false); + }); + }); + } +} + +function addGraph(graph) { + const yScale = graph.yScale(); + + const yAxis = d3.axisLeft().scale(yScale); + + const svg = d3.select("#confusionMatrix").append("svg") + .attr("viewBox", "0, 0," + graph.outerWidth + "," + graph.outerHeight) + .append("g") + .attr("transform", "translate(" + graph.margin.left + "," + graph.margin.top + ")"); + + svg.append("rect") + .attr("class", "outer") + .attr("width", graph.innerWidth) + .attr("height", graph.innerHeight); + + const g = svg.append("g") + .attr("transform", "translate(" + graph.padding.left + "," + graph.padding.top + ")"); + + g.append("g") + .attr("class", "y axis") + .attr("transform", "translate(0,0)") + .call(yAxis); + + g.selectAll("line.horizontalGrid").data(yScale.ticks(13)) + .enter() + .append("line") + .attr("class", "horizontalGrid") + .attr("x1", 0) + .attr("x2", graph.width) + .attr("y1", function(d) { return yScale(d); }) + .attr("y2", function(d) { return yScale(d); }) + .attr("fill", "none") + .attr("shape-rendering", "crispEdges") + .attr("stroke", "steelblue") + .attr("stroke-dasharray", "10,10") + .attr("stroke-width", "1px"); + + g.append("text") + .attr("x", 10) + .attr("y", 15) + .text("True Positive"); + g.append("text") + .attr("x", graph.width - 100) + .attr("y", 15) + .text("False Positive"); + g.append("text") + .attr("x", 10) + .attr("y", graph.height - 5) + .text("False Negative"); + g.append("text") + .attr("x", graph.width - 100) + .attr("y", graph.height - 5) + .text("True Negative"); + g.append("line") + .attr("x1", 0) + .attr("y1", graph.height / 2) + .attr("x2", graph.width) + .attr("y2", graph.height / 2) + .attr("stroke", "steelblue"); + g.append("line") + .attr("x1", graph.width / 2) + .attr("y1", 0) + .attr("x2", graph.width / 2) + .attr("y2", graph.height) + .attr("stroke", "steelblue"); +} + +function getTestResultHtml(test, matrix) { + console.log("getTestResults"); + const t = { + test: { + "<>": "div", + "class": "col-8", + "html": [ + { "<>": "div", "class": "heading", "html": "Model Statistics" }, + { + "<>": "div", + "class": "row", + "html": [ + { "<>": "div", "class": "col-3", "html": "Precision: " + matrix.getPrecision() }, + { "<>": "div", "class": "col-3", "html": "Recall: " + matrix.getRecall() }, + { "<>": "div", "class": "col-3", "html": "F1: " + matrix.getF1() }, + { "<>": "div", "class": "col-3", "html": "Accuracy: " + matrix.getAccuracy() }, + ], + }, + { "<>": "br" }, + { "<>": "div", "class": "heading", "html": "${testName}" }, + { "<>": "p", "html": "Target: ${targetKind}" }, + { "<>": "p", "html": "${resultKind}: ${because}" }, + { + "<>": "div", + "class": "row", + "html": [ + { + "<>": "table", + "class": "table", + "html": [ + { + "<>": "thead", + "html": [ + { + "<>": "tr", + "html": [ + { "<>": "th", "scope": "col", "html": "Expected Utterance" }, + { "<>": "th", "scope": "col", "html": "Actual Utterance" }, + ], + }, + ], + }, + { + "<>": "tbody", + "html": [ + { + "<>": "tr", + "html": [ + { + "<>": "td", + "html"() { + const innerHtml = json2html.transform(this.expectedUtterance, t.utterance); + const actual = false; + return styleEntities(innerHtml, this.expectedUtterance, this.resultKind, actual); + }, + }, + { + "<>": "td", + "html"() { + const innerHtml = json2html.transform(this.actualUtterance, t.utterance); + const actual = true; + return styleEntities(innerHtml, this.actualUtterance, this.resultKind, actual); + }, + }, + ], + }, + ], + }, + ], + }, + ], + }, + ], + }, + utterance: { + "<>": "p", + "html": [ + { "<>": "p", "html": "Text: ${text}" }, + { "<>": "p", "html": "Intent: ${intent}" }, + ], + }, + }; + return json2html.transform(test, t.test); +} + +function styleEntities(html, utterance, resultKind, actualUtterance) { + let styleColor = "rgba(0,255,0,.1)"; // 50% transparent green + if ( + (actualUtterance && (resultKind === "FalsePositive")) || (!actualUtterance && (resultKind === "FalseNegative")) + ) { styleColor = "rgba(255,0,0,.1)"; } // 50% transparent red + if (utterance.entities !== undefined) { + if (utterance.entities.length > 0) { + const searchText = utterance.entities[0].matchText; + html = html.replace(searchText, '' + searchText + ""); + } + } + return html; +} + +function getDotColor(d, i) { + if ((d.resultKind === "TruePositive") || (d.resultKind === "TrueNegative")) { + return "green"; + } + return "red"; +} + +function getXConfusionQuadrant(graph, d, i, matrix) { + const xScale = graph.xScale(); + switch (d.resultKind) { + case "TruePositive": matrix.tp += 1; + break; + case "FalsePositive": matrix.fp += 1; + break; + case "TrueNegative": matrix.tn += 1; + break; + case "FalseNegative": matrix.fn += 1; + break; + } + let x = 0; + if ((d.resultKind === "TruePositive") || (d.resultKind === "FalseNegative")) { + x = xScale(i) + (graph.margin.left + graph.padding.left + 5); + } else { + x = xScale(i + graph.quadrantCountMax) + (graph.margin.left + graph.padding.left + 10); + } + return x; +} + +function getYConfusionQuadrant(graph, d, i) { + const yScale = graph.yScale(); + const value = d.score || 1; + let retVal = 0; + if ((d.resultKind === "TrueNegative") || (d.resultKind === "FalseNegative")) { + const negValue = 0 - value; + retVal = yScale(negValue); + } else { + retVal = yScale(value); + } + return retVal + (graph.margin.top + graph.padding.top); +} diff --git a/extensions/tabs/src/graph.ts b/extensions/tabs/src/graph.ts new file mode 100644 index 0000000..d5f92c3 --- /dev/null +++ b/extensions/tabs/src/graph.ts @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +/* global d3 */ + +// eslint-disable-next-line no-unused-vars +class Graph { + constructor () { + this.margin = { top: 20, right: 20, bottom: 20, left: 20 } + this.padding = { top: 60, right: 60, bottom: 60, left: 60 } + this.outerWidth = 960 + this.outerHeight = 500 + this.innerWidth = this.outerWidth - this.margin.left - this.margin.right + this.innerHeight = this.outerHeight - this.margin.top - this.margin.bottom + this.width = this.innerWidth - this.padding.left - this.padding.right + this.height = this.innerHeight - this.padding.top - this.padding.bottom + this.quadrantCountMax = 0 + } + + yScale () { + return d3.scaleLinear() + .domain([1.2, -1.2]) + .range([0, this.height]) + } + + xScale () { + return d3.scaleLinear() + .domain([0, this.quadrantCountMax * 2]) + .range([0, this.width - 10]) + } +} diff --git a/extensions/tabs/src/init.ts b/extensions/tabs/src/init.ts new file mode 100644 index 0000000..3611eca --- /dev/null +++ b/extensions/tabs/src/init.ts @@ -0,0 +1,45 @@ +/* global VSS, renderConfusionMatrix */ + +// VSS.init({ +// usePlatformScripts: true, +// taskRestClientLoaderConfig: { +// paths: { +// enhancer: 'scripts' +// } +// } +// }) + +// VSS.ready(() => { +// const context = VSS.getWebContext() +// VSS.getConfiguration().onBuildChanged(build => { +// VSS.require(['TFS/DistributedTask/TaskRestClient', 'VSS/Authentication/Services'], (taskRestClient, authServices) => { +// taskRestClient.getClient().getPlanAttachments(context.project.id, 'build', build.orchestrationPlan.planId, 'nlu.devops').then(attachments => { +// let metadata +// let statistics +// attachments.forEach(attachment => { +// if ((attachment.name === 'metadata.json' || attachment.name === 'statistics.json') && attachment._links && attachment._links.self && attachment._links.self.href) { +// metadata = attachment.name === 'metadata.json' ? attachment._links.self.href : metadata +// statistics = attachment.name === 'statistics.json' ? attachment._links.self.href : statistics +// } +// }) + +// if (!metadata) { +// console.warn('Could not find attachment for NLU metadata.') +// } else { +// VSS.getAccessToken().then(token => { +// var authToken = authServices.authTokenManager.getAuthorizationHeader(token) +// renderConfusionMatrix(metadata, { Authorization: authToken }) +// }) +// } + +// if (!statistics) { +// console.warn('Could not find attachment for NLU statistics.') +// } +// }) +// }) +// }) +// }) + +// Metadata for developing charts locally. Not used for production +const metadata = "metadata.json"; +renderConfusionMatrix(metadata, { Authorization: "" }); diff --git a/extensions/tabs/src/matrix.ts b/extensions/tabs/src/matrix.ts new file mode 100644 index 0000000..497b55e --- /dev/null +++ b/extensions/tabs/src/matrix.ts @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// eslint-disable-next-line no-unused-vars +class Matrix { + constructor() { + this.p = 0; + this.n = 0; + this.tp = 0; + this.tn = 0; + this.fp = 0; + this.fn = 0; + } + + public getAccuracy() { + const accuracy = (this.tp + this.tn) / ((this.tp + this.fn) + (this.fp + this.tn)) + return Math.round(accuracy * 100) / 100 + } + + public getF1() { + const f1 = 2 * this.tp / (2 * this.tp + this.fp + this.fn) + return Math.round(f1 * 100) / 100 + } + + public getPrecision() { + const precision = this.tp / (this.tp + this.fp) + return Math.round(precision * 100) / 100 + } + + public getRecall() { + const recall = this.tp / (this.tp + this.fn) + return Math.round(recall * 100) / 100 + } +} diff --git a/extensions/tabs/tsconfig.json b/extensions/tabs/tsconfig.json new file mode 100644 index 0000000..6644f4b --- /dev/null +++ b/extensions/tabs/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "outDir": "dist", + "allowJs": true, + "target": "es5", + "sourceMap": true + }, + "include": [ + "./src/**/*" + ] +} diff --git a/extensions/vss-extension.json b/extensions/vss-extension.json index 74db20a..63ac6fa 100644 --- a/extensions/vss-extension.json +++ b/extensions/vss-extension.json @@ -6,6 +6,7 @@ "description": "A collection of tasks and results tabs for NLU.DevOps", "publisher": "NLUDevOps", "public": true, + "scopes": [ "vso.build" ], "icons": { "default": "images/icon.png" }, @@ -32,6 +33,24 @@ { "path": "tasks/NLUCleanV0", "addressable": true + }, + { + "path": "tabs/NLUResults.html", + "addressable": true + }, + { + "path": "tabs/matrix.css", + "addressable": true + }, + { + "path": "tabs/node_modules/vss-web-extension-sdk/lib", + "addressable": true, + "packagePath": "lib" + }, + { + "path": "tabs/scripts", + "addressable": true, + "packagePath": "scripts" } ], "contributions": [ @@ -64,6 +83,19 @@ "properties": { "name": "tasks/NLUCleanV0" } + }, + { + "id": "nlu-results-tab", + "type": "ms.vss-build-web.build-results-tab", + "description": "A tab for displaying NLU results dependent on the NLUTest task", + "targets": [ + "ms.vss-build-web.build-results-view" + ], + "properties": { + "name": "NLU Results", + "uri": "tabs/NLUResults.html", + "supportsTasks": ["b05f23dd-168e-475a-a6f9-6297f8fd21bc"] + } } ], "content": { diff --git a/pipelines/nlu-extension-build.yml b/pipelines/nlu-extension-build.yml new file mode 100644 index 0000000..71cb3e2 --- /dev/null +++ b/pipelines/nlu-extension-build.yml @@ -0,0 +1,75 @@ +trigger: + branches: + include: + - master + paths: + include: + - /extensions/ + - /pipelines/extension-build.yml +pr: + branches: + include: + - master + paths: + include: + - /extensions/ + - /pipelines/extension-build.yml + +jobs: +- job: build_vsix + pool: + vmImage: 'ubuntu-latest' + steps: + - task: NPM@1 + displayName: npm install + inputs: + command: install + workingDir: extensions + + - task: NPM@1 + displayName: npm run lint + inputs: + command: custom + customCommand: run lint + workingDir: extensions + + - task: NPM@1 + displayName: npm run build + inputs: + command: custom + customCommand: run build + workingDir: extensions + + - task: CopyFiles@2 + condition: ne(variables['isPrivateBuild'], true) + displayName: Stage extension artifact for publish + inputs: + sourceFolder: extensions + contents: '**/*.vsix' + targetFolder: $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + condition: ne(variables['isPrivateBuild'], true) + displayName: Publish extension artifact + inputs: + artifactName: drop + targetPath: $(Build.ArtifactStagingDirectory) + + - task: TfxInstaller@2 + condition: eq(variables['isPrivateBuild'], true) + displayName: Install Node CLI + + - task: PublishAzureDevOpsExtension@2 + displayName: Publish Extension + condition: eq(variables['isPrivateBuild'], true) + inputs: + connectedServiceName: marketplaceConnection + fileType: vsix + vsixFile: '**/*.vsix' + publisherId: NLUDevOps + extensionId: NLUDevOpsCI + extensionName: NLU.DevOps.CI + extensionVersion: '0.1.$(Build.BuildId)' + updateTasksVersion: true + extensionVisibility: private + shareWith: NLUDevOps