From 89f9cda4e3e7f8116ceb728c056624af98258b9d Mon Sep 17 00:00:00 2001 From: Ralf Handl Date: Fri, 1 Nov 2024 11:04:19 +0100 Subject: [PATCH 1/7] Create schema-test-coverage.mjs --- scripts/schema-test-coverage.mjs | 104 +++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 scripts/schema-test-coverage.mjs diff --git a/scripts/schema-test-coverage.mjs b/scripts/schema-test-coverage.mjs new file mode 100644 index 0000000000..e39758b825 --- /dev/null +++ b/scripts/schema-test-coverage.mjs @@ -0,0 +1,104 @@ +import { readdir, readFile } from "node:fs/promises"; +import YAML from "yaml"; +import { join } from "node:path"; +import { argv } from "node:process"; +import "@hyperjump/json-schema/draft-2020-12"; +import { compile, getSchema, interpret, Validation } from "@hyperjump/json-schema/experimental"; +import * as Instance from "@hyperjump/json-schema/instance/experimental"; + +/** + * @import { AST } from "@hyperjump/json-schema/experimental" + * @import { Json } from "@hyperjump/json-schema" + */ + +import contentTypeParser from "content-type"; +import { addMediaTypePlugin } from "@hyperjump/browser"; +import { buildSchemaDocument } from "@hyperjump/json-schema/experimental"; + +addMediaTypePlugin("application/schema+yaml", { + parse: async (response) => { + const contentType = contentTypeParser.parse(response.headers.get("content-type") ?? ""); + const contextDialectId = contentType.parameters.schema ?? contentType.parameters.profile; + + const foo = YAML.parse(await response.text()); + return buildSchemaDocument(foo, response.url, contextDialectId); + }, + fileMatcher: (path) => path.endsWith(".yaml") +}); + +/** @type (testDirectory: string) => AsyncGenerator */ +const tests = async function* (testDirectory) { + for (const file of await readdir(testDirectory, { recursive: true, withFileTypes: true })) { + if (!file.isFile() || !file.name.endsWith(".yaml")) { + continue; + } + + const testPath = join(file.parentPath, file.name); + const testJson = await readFile(testPath, "utf8"); + yield YAML.parse(testJson); + } +}; + +/** @type (testDirectory: string) => Promise */ +const runTests = async (testDirectory) => { + for await (const test of tests(testDirectory)) { + const instance = Instance.fromJs(test); + + interpret(compiled, instance); + } +}; + +/** @type (ast: AST) => string[] */ +const keywordLocations = (ast) => { + /** @type string[] */ + const locations = []; + for (const schemaLocation in ast) { + if (schemaLocation === "metaData") { + continue; + } + + if (Array.isArray(ast[schemaLocation])) { + for (const keyword of ast[schemaLocation]) { + if (Array.isArray(keyword)) { + locations.push(keyword[1]); + } + } + } + } + + return locations; +}; + +/////////////////////////////////////////////////////////////////////////////// + +const schema = await getSchema(argv[2]); +const compiled = await compile(schema); + +/** @type Set */ +const visitedLocations = new Set(); +const baseInterpret = Validation.interpret; +Validation.interpret = (url, instance, ast, dynamicAnchors, quiet) => { + if (Array.isArray(ast[url])) { + for (const keywordNode of ast[url]) { + if (Array.isArray(keywordNode)) { + visitedLocations.add(keywordNode[1]); + } + } + } + return baseInterpret(url, instance, ast, dynamicAnchors, quiet); +}; + +await runTests(argv[3]); +Validation.interpret = baseInterpret; + +// console.log("Covered:", visitedLocations); + +const allKeywords = keywordLocations(compiled.ast); +const notCovered = allKeywords.filter((location) => !visitedLocations.has(location)); +console.log("NOT Covered:", notCovered.length, "of", allKeywords.length,); + +const firstNotCovered = notCovered.slice(0, 20); +if (notCovered.length > 20) firstNotCovered.push("..."); +console.log(firstNotCovered); + +console.log("Covered:", visitedLocations.size, "of", allKeywords.length, "(" + Math.floor(visitedLocations.size / allKeywords.length * 100) + "%)"); From 1951300f3239f65c1e8f387f3422e2dac56c8224 Mon Sep 17 00:00:00 2001 From: Ralf Handl Date: Fri, 1 Nov 2024 11:06:41 +0100 Subject: [PATCH 2/7] Also import draft-04 --- scripts/schema-test-coverage.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/schema-test-coverage.mjs b/scripts/schema-test-coverage.mjs index e39758b825..e92d68f6ff 100644 --- a/scripts/schema-test-coverage.mjs +++ b/scripts/schema-test-coverage.mjs @@ -3,6 +3,7 @@ import YAML from "yaml"; import { join } from "node:path"; import { argv } from "node:process"; import "@hyperjump/json-schema/draft-2020-12"; +import "@hyperjump/json-schema/draft-04"; import { compile, getSchema, interpret, Validation } from "@hyperjump/json-schema/experimental"; import * as Instance from "@hyperjump/json-schema/instance/experimental"; From 1cdc217933945bf273e78b3ca7c5f9bec1e3c0b2 Mon Sep 17 00:00:00 2001 From: Ralf Handl Date: Fri, 1 Nov 2024 11:22:18 +0100 Subject: [PATCH 3/7] test script for schema coverage --- package.json | 2 +- scripts/schema-test-coverage.mjs | 5 +++-- scripts/schema-test-coverage.sh | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 scripts/schema-test-coverage.sh diff --git a/package.json b/package.json index e8cdb13a86..4da52bcd76 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "license": "Apache-2.0", "scripts": { "build": "bash ./scripts/md2html/build.sh", - "test": "c8 --100 vitest --watch=false" + "test": "c8 --100 vitest --watch=false && bash scripts/schema-test-coverage.sh" }, "readmeFilename": "README.md", "files": [ diff --git a/scripts/schema-test-coverage.mjs b/scripts/schema-test-coverage.mjs index e92d68f6ff..1cabe76e87 100644 --- a/scripts/schema-test-coverage.mjs +++ b/scripts/schema-test-coverage.mjs @@ -98,8 +98,9 @@ const allKeywords = keywordLocations(compiled.ast); const notCovered = allKeywords.filter((location) => !visitedLocations.has(location)); console.log("NOT Covered:", notCovered.length, "of", allKeywords.length,); -const firstNotCovered = notCovered.slice(0, 20); -if (notCovered.length > 20) firstNotCovered.push("..."); +const maxNotCovered = 10; +const firstNotCovered = notCovered.slice(0, maxNotCovered); +if (notCovered.length > maxNotCovered) firstNotCovered.push("..."); console.log(firstNotCovered); console.log("Covered:", visitedLocations.size, "of", allKeywords.length, "(" + Math.floor(visitedLocations.size / allKeywords.length * 100) + "%)"); diff --git a/scripts/schema-test-coverage.sh b/scripts/schema-test-coverage.sh new file mode 100644 index 0000000000..53fb8f3eb0 --- /dev/null +++ b/scripts/schema-test-coverage.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# Author: @ralfhandl + +# Run this script from the root of the repo + +echo +echo "Schema Test Coverage" +echo + +for schemaDir in schemas/v3* ; do + version=$(basename "$schemaDir") + echo $version + + node scripts/schema-test-coverage.mjs $schemaDir/schema.yaml tests/$version + + echo +done \ No newline at end of file From f9cd70a34ea41ab9d9cd6f4285100e4c8eb95e13 Mon Sep 17 00:00:00 2001 From: Ralf Handl Date: Fri, 1 Nov 2024 20:54:48 +0100 Subject: [PATCH 4/7] Use only pass cases for coverage --- scripts/schema-test-coverage.mjs | 6 ++++-- scripts/schema-test-coverage.sh | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/schema-test-coverage.mjs b/scripts/schema-test-coverage.mjs index 1cabe76e87..88f9a97cb1 100644 --- a/scripts/schema-test-coverage.mjs +++ b/scripts/schema-test-coverage.mjs @@ -4,7 +4,7 @@ import { join } from "node:path"; import { argv } from "node:process"; import "@hyperjump/json-schema/draft-2020-12"; import "@hyperjump/json-schema/draft-04"; -import { compile, getSchema, interpret, Validation } from "@hyperjump/json-schema/experimental"; +import { compile, getSchema, interpret, Validation, BASIC } from "@hyperjump/json-schema/experimental"; import * as Instance from "@hyperjump/json-schema/instance/experimental"; /** @@ -45,7 +45,9 @@ const runTests = async (testDirectory) => { for await (const test of tests(testDirectory)) { const instance = Instance.fromJs(test); - interpret(compiled, instance); + const result = interpret(compiled, instance, BASIC); + //TODO: now result has errors array if valid is false + // if (!result.valid) console.log(result) } }; diff --git a/scripts/schema-test-coverage.sh b/scripts/schema-test-coverage.sh index 53fb8f3eb0..0e4ed3883e 100644 --- a/scripts/schema-test-coverage.sh +++ b/scripts/schema-test-coverage.sh @@ -12,7 +12,7 @@ for schemaDir in schemas/v3* ; do version=$(basename "$schemaDir") echo $version - node scripts/schema-test-coverage.mjs $schemaDir/schema.yaml tests/$version + node scripts/schema-test-coverage.mjs $schemaDir/schema.yaml tests/$version/pass echo done \ No newline at end of file From ff0fdece0ebd0f275cc90a9eee83b1f3a6860059 Mon Sep 17 00:00:00 2001 From: Ralf Handl Date: Tue, 12 Nov 2024 11:29:51 +0100 Subject: [PATCH 5/7] Show validation errors if test instances --- scripts/schema-test-coverage.mjs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/scripts/schema-test-coverage.mjs b/scripts/schema-test-coverage.mjs index 88f9a97cb1..8654200ddc 100644 --- a/scripts/schema-test-coverage.mjs +++ b/scripts/schema-test-coverage.mjs @@ -27,7 +27,7 @@ addMediaTypePlugin("application/schema+yaml", { fileMatcher: (path) => path.endsWith(".yaml") }); -/** @type (testDirectory: string) => AsyncGenerator */ +/** @type (testDirectory: string) => AsyncGenerator<[string,Json]> */ const tests = async function* (testDirectory) { for (const file of await readdir(testDirectory, { recursive: true, withFileTypes: true })) { if (!file.isFile() || !file.name.endsWith(".yaml")) { @@ -36,18 +36,21 @@ const tests = async function* (testDirectory) { const testPath = join(file.parentPath, file.name); const testJson = await readFile(testPath, "utf8"); - yield YAML.parse(testJson); + + yield [testPath, YAML.parse(testJson)]; } }; /** @type (testDirectory: string) => Promise */ const runTests = async (testDirectory) => { - for await (const test of tests(testDirectory)) { + for await (const [name, test] of tests(testDirectory)) { const instance = Instance.fromJs(test); const result = interpret(compiled, instance, BASIC); - //TODO: now result has errors array if valid is false - // if (!result.valid) console.log(result) + + if (!result.valid) { + console.log("Failed:", name, result.errors); + } } }; @@ -100,7 +103,7 @@ const allKeywords = keywordLocations(compiled.ast); const notCovered = allKeywords.filter((location) => !visitedLocations.has(location)); console.log("NOT Covered:", notCovered.length, "of", allKeywords.length,); -const maxNotCovered = 10; +const maxNotCovered = 20; const firstNotCovered = notCovered.slice(0, maxNotCovered); if (notCovered.length > maxNotCovered) firstNotCovered.push("..."); console.log(firstNotCovered); From 1f2a389b619bf9425bd92b547355e82ab91dc0b5 Mon Sep 17 00:00:00 2001 From: Ralf Handl Date: Tue, 12 Nov 2024 12:51:33 +0100 Subject: [PATCH 6/7] Prettier --- scripts/schema-test-coverage.mjs | 49 +++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/scripts/schema-test-coverage.mjs b/scripts/schema-test-coverage.mjs index 8654200ddc..9953a5c291 100644 --- a/scripts/schema-test-coverage.mjs +++ b/scripts/schema-test-coverage.mjs @@ -4,7 +4,13 @@ import { join } from "node:path"; import { argv } from "node:process"; import "@hyperjump/json-schema/draft-2020-12"; import "@hyperjump/json-schema/draft-04"; -import { compile, getSchema, interpret, Validation, BASIC } from "@hyperjump/json-schema/experimental"; +import { + compile, + getSchema, + interpret, + Validation, + BASIC, +} from "@hyperjump/json-schema/experimental"; import * as Instance from "@hyperjump/json-schema/instance/experimental"; /** @@ -18,18 +24,24 @@ import { buildSchemaDocument } from "@hyperjump/json-schema/experimental"; addMediaTypePlugin("application/schema+yaml", { parse: async (response) => { - const contentType = contentTypeParser.parse(response.headers.get("content-type") ?? ""); - const contextDialectId = contentType.parameters.schema ?? contentType.parameters.profile; + const contentType = contentTypeParser.parse( + response.headers.get("content-type") ?? "", + ); + const contextDialectId = + contentType.parameters.schema ?? contentType.parameters.profile; const foo = YAML.parse(await response.text()); return buildSchemaDocument(foo, response.url, contextDialectId); }, - fileMatcher: (path) => path.endsWith(".yaml") + fileMatcher: (path) => path.endsWith(".yaml"), }); /** @type (testDirectory: string) => AsyncGenerator<[string,Json]> */ const tests = async function* (testDirectory) { - for (const file of await readdir(testDirectory, { recursive: true, withFileTypes: true })) { + for (const file of await readdir(testDirectory, { + recursive: true, + withFileTypes: true, + })) { if (!file.isFile() || !file.name.endsWith(".yaml")) { continue; } @@ -100,12 +112,21 @@ Validation.interpret = baseInterpret; // console.log("Covered:", visitedLocations); const allKeywords = keywordLocations(compiled.ast); -const notCovered = allKeywords.filter((location) => !visitedLocations.has(location)); -console.log("NOT Covered:", notCovered.length, "of", allKeywords.length,); - -const maxNotCovered = 20; -const firstNotCovered = notCovered.slice(0, maxNotCovered); -if (notCovered.length > maxNotCovered) firstNotCovered.push("..."); -console.log(firstNotCovered); - -console.log("Covered:", visitedLocations.size, "of", allKeywords.length, "(" + Math.floor(visitedLocations.size / allKeywords.length * 100) + "%)"); +const notCovered = allKeywords.filter( + (location) => !visitedLocations.has(location), +); +if (notCovered.length > 0) { + console.log("NOT Covered:", notCovered.length, "of", allKeywords.length); + const maxNotCovered = 20; + const firstNotCovered = notCovered.slice(0, maxNotCovered); + if (notCovered.length > maxNotCovered) firstNotCovered.push("..."); + console.log(firstNotCovered); +} + +console.log( + "Covered:", + visitedLocations.size, + "of", + allKeywords.length, + "(" + Math.floor((visitedLocations.size / allKeywords.length) * 100) + "%)", +); From 31f66e72f5708e85055bdc70c04763ca3238d721 Mon Sep 17 00:00:00 2001 From: Ralf Handl Date: Tue, 12 Nov 2024 12:56:59 +0100 Subject: [PATCH 7/7] make executable --- scripts/schema-test-coverage.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/schema-test-coverage.sh diff --git a/scripts/schema-test-coverage.sh b/scripts/schema-test-coverage.sh old mode 100644 new mode 100755