From 61575838f333fffcb1b884826fa5a09fcc7b94bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Berg=C3=A9?= Date: Wed, 19 Feb 2025 10:29:33 +0100 Subject: [PATCH] Optimize markdown parsing (#2850) --- .changeset/nervous-buses-own.md | 7 + bun.lock | 32 ++--- packages/gitbook/package.json | 8 +- .../DocumentView/OpenAPI/OpenAPI.tsx | 2 +- .../lib/__snapshots__/markdown.test.ts.snap | 133 ++++++++++++++++++ packages/gitbook/src/lib/document-sections.ts | 2 +- packages/gitbook/src/lib/markdown.test.ts | 4 +- packages/gitbook/src/lib/markdown.ts | 24 +--- .../gitbook/src/lib/openapi/enrich.test.ts | 19 +++ packages/gitbook/src/lib/openapi/enrich.ts | 43 ++++++ .../src/lib/{openapi.ts => openapi/fetch.ts} | 37 +---- .../lib/openapi/fixtures/multiline-spec.yaml | 41 ++++++ packages/openapi-parser/src/types.ts | 4 +- .../src/resolveOpenAPIOperation.test.ts | 2 +- packages/react-openapi/src/utils.ts | 5 +- 15 files changed, 283 insertions(+), 80 deletions(-) create mode 100644 .changeset/nervous-buses-own.md create mode 100644 packages/gitbook/src/lib/__snapshots__/markdown.test.ts.snap create mode 100644 packages/gitbook/src/lib/openapi/enrich.test.ts create mode 100644 packages/gitbook/src/lib/openapi/enrich.ts rename packages/gitbook/src/lib/{openapi.ts => openapi/fetch.ts} (64%) create mode 100644 packages/gitbook/src/lib/openapi/fixtures/multiline-spec.yaml diff --git a/.changeset/nervous-buses-own.md b/.changeset/nervous-buses-own.md new file mode 100644 index 000000000..1e8b62051 --- /dev/null +++ b/.changeset/nervous-buses-own.md @@ -0,0 +1,7 @@ +--- +'@gitbook/openapi-parser': patch +'@gitbook/react-openapi': patch +'gitbook': patch +--- + +Improve Markdown parsing diff --git a/bun.lock b/bun.lock index 952eba438..d0591adc2 100644 --- a/bun.lock +++ b/bun.lock @@ -31,7 +31,7 @@ }, "packages/gitbook": { "name": "gitbook", - "version": "0.6.0", + "version": "0.6.1", "dependencies": { "@gitbook/api": "^0.93.0", "@gitbook/cache-do": "workspace:*", @@ -61,6 +61,8 @@ "mathjax": "^3.2.2", "mdast-util-to-markdown": "^2.1.2", "memoizee": "^0.4.15", + "micromark": "^4.0.1", + "micromark-extension-gfm": "^3.0.0", "next": "14.2.23", "next-themes": "^0.2.1", "nuqs": "^2.2.3", @@ -71,17 +73,11 @@ "react": "18.3.1", "react-dom": "18.3.1", "react-hotkeys-hook": "^4.4.1", - "rehype-sanitize": "^6.0.0", - "rehype-stringify": "^10.0.0", - "remark-gfm": "^4.0.0", - "remark-parse": "^11.0.0", - "remark-rehype": "^11.1.0", "rison": "^0.1.1", "server-only": "^0.0.1", "shiki": "^1.27.2", "tailwind-merge": "^2.2.0", "tailwind-shades": "^1.1.2", - "unified": "^11.0.4", "url-join": "^5.0.0", "usehooks-ts": "^3.1.0", }, @@ -207,7 +203,7 @@ }, "packages/react-openapi": { "name": "@gitbook/react-openapi", - "version": "1.0.0", + "version": "1.0.1", "dependencies": { "@gitbook/openapi-parser": "workspace:*", "@scalar/api-client-react": "1.0.87", @@ -2210,7 +2206,7 @@ "hast-util-sanitize": ["hast-util-sanitize@5.0.1", "", { "dependencies": { "@types/hast": "^3.0.0", "@ungap/structured-clone": "^1.2.0", "unist-util-position": "^5.0.0" } }, "sha512-IGrgWLuip4O2nq5CugXy4GI2V8kx4sFVy5Hd4vF7AR2gxS0N9s7nEAVUyeMtZKZvzrxVsHt73XdTsno1tClIkQ=="], - "hast-util-to-html": ["hast-util-to-html@9.0.3", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/unist": "^3.0.0", "ccount": "^2.0.0", "comma-separated-tokens": "^2.0.0", "hast-util-whitespace": "^3.0.0", "html-void-elements": "^3.0.0", "mdast-util-to-hast": "^13.0.0", "property-information": "^6.0.0", "space-separated-tokens": "^2.0.0", "stringify-entities": "^4.0.0", "zwitch": "^2.0.4" } }, "sha512-M17uBDzMJ9RPCqLMO92gNNUDuBSq10a25SDBI08iCCxmorf4Yy6sYHK57n9WAbRAAaU+DuR4W6GN9K4DFZesYg=="], + "hast-util-to-html": ["hast-util-to-html@9.0.4", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/unist": "^3.0.0", "ccount": "^2.0.0", "comma-separated-tokens": "^2.0.0", "hast-util-whitespace": "^3.0.0", "html-void-elements": "^3.0.0", "mdast-util-to-hast": "^13.0.0", "property-information": "^6.0.0", "space-separated-tokens": "^2.0.0", "stringify-entities": "^4.0.0", "zwitch": "^2.0.4" } }, "sha512-wxQzXtdbhiwGAUKrnQJXlOPmHnEehzphwkK7aluUPQ+lEc1xefC8pblMgpp2w5ldBTEfveRIrADcrhGIWrlTDA=="], "hast-util-to-parse5": ["hast-util-to-parse5@8.0.0", "", { "dependencies": { "@types/hast": "^3.0.0", "comma-separated-tokens": "^2.0.0", "devlop": "^1.0.0", "property-information": "^6.0.0", "space-separated-tokens": "^2.0.0", "web-namespaces": "^2.0.0", "zwitch": "^2.0.0" } }, "sha512-3KKrV5ZVI8if87DVSi1vDeByYrkGzg4mEfeu4alwgmmIeARiBLKCZS2uw5Gb6nU9x9Yufyj3iudm6i7nl52PFw=="], @@ -2332,7 +2328,7 @@ "is-path-inside": ["is-path-inside@3.0.3", "", {}, "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ=="], - "is-plain-obj": ["is-plain-obj@4.1.0", "", {}, "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg=="], + "is-plain-obj": ["is-plain-obj@1.1.0", "", {}, "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg=="], "is-promise": ["is-promise@2.2.2", "", {}, "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ=="], @@ -2542,7 +2538,7 @@ "microdiff": ["microdiff@1.4.0", "", {}, "sha512-OBKBOa1VBznvLPb/3ljeJaENVe0fO0lnWl77lR4vhPlQD71UpjEoRV5P0KdQkcjbFlBu1Oy2mEUBMU3wxcBAGg=="], - "micromark": ["micromark@4.0.0", "", { "dependencies": { "@types/debug": "^4.0.0", "debug": "^4.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-encode": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-o/sd0nMof8kYff+TqcDx3VSrgBTcZpSvYcAHIfHhv5VAuNmisCxjhx6YmxS8PFEpb9z5WKWKPdzf0jM23ro3RQ=="], + "micromark": ["micromark@4.0.1", "", { "dependencies": { "@types/debug": "^4.0.0", "debug": "^4.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-encode": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-eBPdkcoCNvYcxQOAKAlceo5SNdzZWfF+FcSupREAzdAh9rRmE239CEQAiTwIgblwnoM8zzj35sZ5ZwvSEOF6Kw=="], "micromark-core-commonmark": ["micromark-core-commonmark@2.0.1", "", { "dependencies": { "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-factory-destination": "^2.0.0", "micromark-factory-label": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-factory-title": "^2.0.0", "micromark-factory-whitespace": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-html-tag-name": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-CUQyKr1e///ZODyD1U3xit6zXwy1a8q2a1S1HKtIlmgvurrEpaw/Y9y6KSIbF8P59cn/NjzHyO+Q2fAyYLQrAA=="], @@ -4292,8 +4288,6 @@ "@sentry/webpack-plugin/uuid": ["uuid@9.0.1", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA=="], - "@shikijs/core/hast-util-to-html": ["hast-util-to-html@9.0.4", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/unist": "^3.0.0", "ccount": "^2.0.0", "comma-separated-tokens": "^2.0.0", "hast-util-whitespace": "^3.0.0", "html-void-elements": "^3.0.0", "mdast-util-to-hast": "^13.0.0", "property-information": "^6.0.0", "space-separated-tokens": "^2.0.0", "stringify-entities": "^4.0.0", "zwitch": "^2.0.4" } }, "sha512-wxQzXtdbhiwGAUKrnQJXlOPmHnEehzphwkK7aluUPQ+lEc1xefC8pblMgpp2w5ldBTEfveRIrADcrhGIWrlTDA=="], - "@smithy/abort-controller/tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], "@smithy/chunked-blob-reader/tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], @@ -4682,6 +4676,8 @@ "mdast-util-find-and-replace/escape-string-regexp": ["escape-string-regexp@5.0.0", "", {}, "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw=="], + "mdast-util-from-markdown/micromark": ["micromark@4.0.0", "", { "dependencies": { "@types/debug": "^4.0.0", "debug": "^4.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-encode": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-o/sd0nMof8kYff+TqcDx3VSrgBTcZpSvYcAHIfHhv5VAuNmisCxjhx6YmxS8PFEpb9z5WKWKPdzf0jM23ro3RQ=="], + "mdast-util-gfm/mdast-util-to-markdown": ["mdast-util-to-markdown@2.1.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "longest-streak": "^3.0.0", "mdast-util-phrasing": "^4.0.0", "mdast-util-to-string": "^4.0.0", "micromark-util-decode-string": "^2.0.0", "unist-util-visit": "^5.0.0", "zwitch": "^2.0.0" } }, "sha512-SR2VnIEdVNCJbP6y7kVTJgPLifdr8WEU440fQec7qHoHOUz/oJ2jmNRqdDQ3rbiStOXb2mCDGTuwsK5OPUgYlQ=="], "mdast-util-gfm-footnote/mdast-util-to-markdown": ["mdast-util-to-markdown@2.1.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "longest-streak": "^3.0.0", "mdast-util-phrasing": "^4.0.0", "mdast-util-to-string": "^4.0.0", "micromark-util-decode-string": "^2.0.0", "unist-util-visit": "^5.0.0", "zwitch": "^2.0.0" } }, "sha512-SR2VnIEdVNCJbP6y7kVTJgPLifdr8WEU440fQec7qHoHOUz/oJ2jmNRqdDQ3rbiStOXb2mCDGTuwsK5OPUgYlQ=="], @@ -4700,12 +4696,8 @@ "micro/content-type": ["content-type@1.0.4", "", {}, "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA=="], - "micromark/debug": ["debug@4.3.7", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ=="], - "minimist-options/arrify": ["arrify@1.0.1", "", {}, "sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA=="], - "minimist-options/is-plain-obj": ["is-plain-obj@1.1.0", "", {}, "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg=="], - "minizlib/minipass": ["minipass@2.9.0", "", { "dependencies": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" } }, "sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg=="], "next/@swc/helpers": ["@swc/helpers@0.5.5", "", { "dependencies": { "@swc/counter": "^0.1.3", "tslib": "^2.4.0" } }, "sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A=="], @@ -4760,6 +4752,8 @@ "read-yaml-file/pify": ["pify@4.0.1", "", {}, "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g=="], + "rehype-stringify/hast-util-to-html": ["hast-util-to-html@9.0.3", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/unist": "^3.0.0", "ccount": "^2.0.0", "comma-separated-tokens": "^2.0.0", "hast-util-whitespace": "^3.0.0", "html-void-elements": "^3.0.0", "mdast-util-to-hast": "^13.0.0", "property-information": "^6.0.0", "space-separated-tokens": "^2.0.0", "stringify-entities": "^4.0.0", "zwitch": "^2.0.4" } }, "sha512-M17uBDzMJ9RPCqLMO92gNNUDuBSq10a25SDBI08iCCxmorf4Yy6sYHK57n9WAbRAAaU+DuR4W6GN9K4DFZesYg=="], + "remark-stringify/mdast-util-to-markdown": ["mdast-util-to-markdown@2.1.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "longest-streak": "^3.0.0", "mdast-util-phrasing": "^4.0.0", "mdast-util-to-string": "^4.0.0", "micromark-util-decode-string": "^2.0.0", "unist-util-visit": "^5.0.0", "zwitch": "^2.0.0" } }, "sha512-SR2VnIEdVNCJbP6y7kVTJgPLifdr8WEU440fQec7qHoHOUz/oJ2jmNRqdDQ3rbiStOXb2mCDGTuwsK5OPUgYlQ=="], "require-in-the-middle/debug": ["debug@4.3.7", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ=="], @@ -4814,6 +4808,8 @@ "type-is/mime-types": ["mime-types@3.0.0", "", { "dependencies": { "mime-db": "^1.53.0" } }, "sha512-XqoSHeCGjVClAmoGFG3lVFqQFRIrTVw2OH3axRqAcfaw+gHWIfnASS92AV+Rl/mk0MupgZTRHQOjxY6YVnzK5w=="], + "unified/is-plain-obj": ["is-plain-obj@4.1.0", "", {}, "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg=="], + "unplugin/chokidar": ["chokidar@3.6.0", "", { "dependencies": { "anymatch": "~3.1.2", "braces": "~3.0.2", "glob-parent": "~5.1.2", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", "normalize-path": "~3.0.0", "readdirp": "~3.6.0" }, "optionalDependencies": { "fsevents": "~2.3.2" } }, "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw=="], "update-notifier/chalk": ["chalk@3.0.0", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg=="], @@ -5614,6 +5610,8 @@ "gtoken/jws/jwa": ["jwa@2.0.0", "", { "dependencies": { "buffer-equal-constant-time": "1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA=="], + "mdast-util-from-markdown/micromark/debug": ["debug@4.3.7", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ=="], + "raw-body/http-errors/depd": ["depd@1.1.2", "", {}, "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ=="], "raw-body/http-errors/inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], diff --git a/packages/gitbook/package.json b/packages/gitbook/package.json index 228388214..f7f5dc099 100644 --- a/packages/gitbook/package.json +++ b/packages/gitbook/package.json @@ -45,6 +45,8 @@ "mathjax": "^3.2.2", "mdast-util-to-markdown": "^2.1.2", "memoizee": "^0.4.15", + "micromark": "^4.0.1", + "micromark-extension-gfm": "^3.0.0", "next": "14.2.23", "next-themes": "^0.2.1", "nuqs": "^2.2.3", @@ -55,17 +57,11 @@ "react": "18.3.1", "react-dom": "18.3.1", "react-hotkeys-hook": "^4.4.1", - "rehype-sanitize": "^6.0.0", - "rehype-stringify": "^10.0.0", - "remark-gfm": "^4.0.0", - "remark-parse": "^11.0.0", - "remark-rehype": "^11.1.0", "rison": "^0.1.1", "server-only": "^0.0.1", "shiki": "^1.27.2", "tailwind-merge": "^2.2.0", "tailwind-shades": "^1.1.2", - "unified": "^11.0.4", "url-join": "^5.0.0", "usehooks-ts": "^3.1.0" }, diff --git a/packages/gitbook/src/components/DocumentView/OpenAPI/OpenAPI.tsx b/packages/gitbook/src/components/DocumentView/OpenAPI/OpenAPI.tsx index e92d680c8..7f1c3b2f9 100644 --- a/packages/gitbook/src/components/DocumentView/OpenAPI/OpenAPI.tsx +++ b/packages/gitbook/src/components/DocumentView/OpenAPI/OpenAPI.tsx @@ -4,7 +4,7 @@ import { OpenAPIOperation } from '@gitbook/react-openapi'; import React from 'react'; import { LoadingPane } from '@/components/primitives'; -import { fetchOpenAPIBlock } from '@/lib/openapi'; +import { fetchOpenAPIBlock } from '@/lib/openapi/fetch'; import { tcls } from '@/lib/tailwind'; import { BlockProps } from '../Block'; diff --git a/packages/gitbook/src/lib/__snapshots__/markdown.test.ts.snap b/packages/gitbook/src/lib/__snapshots__/markdown.test.ts.snap new file mode 100644 index 000000000..4f901b2a5 --- /dev/null +++ b/packages/gitbook/src/lib/__snapshots__/markdown.test.ts.snap @@ -0,0 +1,133 @@ +// Bun Snapshot v1, https://goo.gl/fbAQLP + +exports[`parseMarkdown should parse a simple table 1`] = ` +"

Table

+ + + + + + + + + +
abcd
" +`; + +exports[`parseMarkdown should parse a complex table 1`] = ` +"

Returns information for all non-fungible tokens for an account.

+

Ordering

+

When considering NFTs, their order is governed by a combination of their numerical token.Id and serialnumber values, with token.id being the parent column. +A serialnumbers value governs its order within the given token.id

+

In that regard, if a user acquired a set of NFTs in the order (2-2, 2-4 1-5, 1-1, 1-3, 3-3, 3-4), the following layouts illustrate the ordering expectations for ownership listing

+
    +
  1. All NFTs in ASC order: 1-1, 1-3, 1-5, 2-2, 2-4, 3-3, 3-4
    2. +
  2. All NFTs in DESC order: 3-4, 3-3, 2-4, 2-2, 1-5, 1-3, 1-1
    3. +
  3. NFTs above 1-1 in ASC order: 1-3, 1-5, 2-2, 2-4, 3-3, 3-4
    4. +
  4. NFTs below 3-3 in ASC order: 1-1, 1-3, 1-5, 2-2, 2-4
    5. +
  5. NFTs between 1-3 and 3-3 inclusive in DESC order: 3-4, 3-3, 2-4, 2-2, 1-5, 1-3
    6. +
+

Note: The default order for this API is currently DESC

+

Filtering

+

When filtering there are some restrictions enforced to ensure correctness and scalability.

+

The table below defines the restrictions and support for the NFT ownership endpoint

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Query ParamComparison OperatorSupportDescriptionExample
token.ideqYSingle occurrence only.?token.id=X
neN
lt(e)YSingle occurrence only.?token.id=lte:X
gt(e)YSingle occurrence only.?token.id=gte:X
serialnumbereqYSingle occurrence only. Requires the presence of a token.id query?serialnumber=Y
neN
lt(e)YSingle occurrence only. Requires the presence of an lte or eq token.id query?token.id=lte:X&serialnumber=lt:Y
gt(e)YSingle occurrence only. Requires the presence of an gte or eq token.id query?token.id=gte:X&serialnumber=gt:Y
spender.ideqY?spender.id=Z
neN
lt(e)Y?spender.id=lt:Z
gt(e)Y?spender.id=gt:Z
+

Note: When searching across a range for individual NFTs a serialnumber with an additional token.id query filter must be provided. +Both filters must be a single occurrence of gt(e) or lt(e) which provide a lower and or upper boundary for search.

" +`; diff --git a/packages/gitbook/src/lib/document-sections.ts b/packages/gitbook/src/lib/document-sections.ts index 51abac2b9..375b5812d 100644 --- a/packages/gitbook/src/lib/document-sections.ts +++ b/packages/gitbook/src/lib/document-sections.ts @@ -1,7 +1,7 @@ import { JSONDocument, ContentRef } from '@gitbook/api'; import { getNodeText } from './document'; -import { fetchOpenAPIBlock } from './openapi'; +import { fetchOpenAPIBlock } from './openapi/fetch'; import { ResolvedContentRef } from './references'; export interface DocumentSection { diff --git a/packages/gitbook/src/lib/markdown.test.ts b/packages/gitbook/src/lib/markdown.test.ts index 4983fa2d3..c59bd6aaf 100644 --- a/packages/gitbook/src/lib/markdown.test.ts +++ b/packages/gitbook/src/lib/markdown.test.ts @@ -9,7 +9,7 @@ describe('parseMarkdown', () => { | a | b | c | d | | - | :- | -: | :-: |`); - expect(result).toContain(''); + expect(result).toMatchSnapshot(); }); it('should parse a complex table', async () => { @@ -52,6 +52,6 @@ When filtering there are some restrictions enforced to ensure correctness and sc Note: When searching across a range for individual NFTs a **serialnumber** with an additional **token.id** query filter must be provided. Both filters must be a single occurrence of **gt(e)** or **lt(e)** which provide a lower and or upper boundary for search.`); - expect(result).toContain('
'); + expect(result).toMatchSnapshot(); }); }); diff --git a/packages/gitbook/src/lib/markdown.ts b/packages/gitbook/src/lib/markdown.ts index f60208207..3201277b4 100644 --- a/packages/gitbook/src/lib/markdown.ts +++ b/packages/gitbook/src/lib/markdown.ts @@ -1,22 +1,12 @@ -import rehypeSanitize from 'rehype-sanitize'; -import rehypeStringify from 'rehype-stringify'; -import remarkGfm from 'remark-gfm'; -import remarkParse from 'remark-parse'; -import remarkRehype from 'remark-rehype'; -import { unified } from 'unified'; +import { micromark } from 'micromark'; +import { gfmHtml, gfm } from 'micromark-extension-gfm'; /** * Parse markdown and output HTML. */ -export async function parseMarkdown(markdown: string): Promise { - const promise = unified() - .use(remarkParse) - .use(remarkGfm) - .use(remarkRehype) - .use(rehypeSanitize) - .use(rehypeStringify) - .process(markdown) - .then((file) => file.toString()); - - return promise; +export function parseMarkdown(input: string): string { + return micromark(input, { + extensions: [gfm()], + htmlExtensions: [gfmHtml()], + }); } diff --git a/packages/gitbook/src/lib/openapi/enrich.test.ts b/packages/gitbook/src/lib/openapi/enrich.test.ts new file mode 100644 index 000000000..3faaed390 --- /dev/null +++ b/packages/gitbook/src/lib/openapi/enrich.test.ts @@ -0,0 +1,19 @@ +import { parseOpenAPI } from '@gitbook/openapi-parser'; +import { describe, expect, it } from 'bun:test'; + +import { enrichFilesystem } from './enrich'; + +const spec = await Bun.file(new URL('./fixtures/multiline-spec.yaml', import.meta.url)).text(); + +describe('#enrichFilesystem', () => { + it('supports multiline descriptions', async () => { + const filesystem = await parseOpenAPI({ + value: spec, + rootURL: null, + }); + const enriched = await enrichFilesystem(filesystem); + expect(enriched[0].specification.paths['/pet'].put['x-gitbook-description-html']).toBe( + '

Social platform

', + ); + }); +}); diff --git a/packages/gitbook/src/lib/openapi/enrich.ts b/packages/gitbook/src/lib/openapi/enrich.ts new file mode 100644 index 000000000..b76dbc9db --- /dev/null +++ b/packages/gitbook/src/lib/openapi/enrich.ts @@ -0,0 +1,43 @@ +import { traverse, Filesystem } from '@gitbook/openapi-parser'; + +import { parseMarkdown } from '../markdown'; + +/** + * Enrich a filesystem with HTML descriptions. + */ +export async function enrichFilesystem(filesystem: Filesystem) { + const parseMarkdownWithCache = createMarkdownParser(); + return traverse(filesystem, async (node, path) => { + if ( + path !== undefined && + 'description' in node && + typeof node.description === 'string' && + node.description + ) { + const key = path[path.length - 1]; + // Avoid parsing descriptions in examples. + if (key !== 'example') { + const description = node.description.trim(); + if (description) { + node['x-gitbook-description-html'] = parseMarkdownWithCache(description); + } + } + } + return node; + }); +} + +/** + * Create a markdown parser that caches the results of parsing. + */ +const createMarkdownParser = () => (input: string) => { + const cache: Record = {}; + const existing = cache[input]; + if (existing) { + return existing; + } + + const result = parseMarkdown(input); + cache[input] = result; + return result; +}; diff --git a/packages/gitbook/src/lib/openapi.ts b/packages/gitbook/src/lib/openapi/fetch.ts similarity index 64% rename from packages/gitbook/src/lib/openapi.ts rename to packages/gitbook/src/lib/openapi/fetch.ts index e089b3097..afb5f2f5b 100644 --- a/packages/gitbook/src/lib/openapi.ts +++ b/packages/gitbook/src/lib/openapi/fetch.ts @@ -1,11 +1,11 @@ import { ContentRef, DocumentBlockOpenAPI } from '@gitbook/api'; -import { parseOpenAPI, OpenAPIParseError, traverse } from '@gitbook/openapi-parser'; +import { parseOpenAPI, OpenAPIParseError } from '@gitbook/openapi-parser'; import { type OpenAPIOperationData, resolveOpenAPIOperation } from '@gitbook/react-openapi'; import { cache, noCacheFetchOptions, CacheFunctionOptions } from '@/lib/cache'; -import { parseMarkdown } from './markdown'; -import { ResolvedContentRef } from './references'; +import { enrichFilesystem } from './enrich'; +import { ResolvedContentRef } from '../references'; /** * Fetch an OpenAPI specification for an operation. @@ -40,7 +40,7 @@ export async function fetchOpenAPIBlock( } const fetchFilesystem = cache({ - name: 'openapi.fetch.v5', + name: 'openapi.fetch.v6', get: async (url: string, options: CacheFunctionOptions) => { // Wrap the raw string to prevent invalid URLs from being passed to fetch. // This can happen if the URL has whitespace, which is currently handled differently by Cloudflare's implementation of fetch: @@ -58,38 +58,13 @@ const fetchFilesystem = cache({ const text = await response.text(); const filesystem = await parseOpenAPI({ value: text, rootURL: url }); - const parseMarkdownWithCache = createMarkdownParser(); - const transformedFs = await traverse(filesystem, async (node, path) => { - if ('description' in node && typeof node.description === 'string' && node.description) { - const lastKey = path && path[path.length - 1]; - // Avoid parsing descriptions in examples. - if (lastKey === 'example') { - return node; - } - node['x-description-html'] = await parseMarkdownWithCache(node.description); - } - return node; - }); + const richFilesystem = await enrichFilesystem(filesystem); return { // Cache for 4 hours ttl: 24 * 60 * 60, // Revalidate every 2 hours revalidateBefore: 22 * 60 * 60, - data: transformedFs, + data: richFilesystem, }; }, }); - -/** - * Create a markdown parser that caches the results of parsing. - */ -const createMarkdownParser = () => async (input: string) => { - const cache = new Map>(); - if (cache.has(input)) { - return cache.get(input) as Promise; - } - - const promise = parseMarkdown(input); - cache.set(input, promise); - return promise; -}; diff --git a/packages/gitbook/src/lib/openapi/fixtures/multiline-spec.yaml b/packages/gitbook/src/lib/openapi/fixtures/multiline-spec.yaml new file mode 100644 index 000000000..70aefdc96 --- /dev/null +++ b/packages/gitbook/src/lib/openapi/fixtures/multiline-spec.yaml @@ -0,0 +1,41 @@ +openapi: 3.0.2 +info: + title: OpenAPI spec + version: 0.0.0 +servers: + - url: '/api/v3' +paths: + '/pet': + put: + summary: Update an existing pet + description: | + Social platform + operationId: updatePet + requestBody: + description: Update an existent pet in the store + content: + application/json: + schema: + required: + - id + type: object + properties: + id: + type: integer + format: int64 + example: 10 + required: true + responses: + '200': + description: Successful operation + content: + application/json: + schema: + required: + - id + type: object + properties: + id: + type: integer + format: int64 + example: 10 diff --git a/packages/openapi-parser/src/types.ts b/packages/openapi-parser/src/types.ts index 218540511..560053a07 100644 --- a/packages/openapi-parser/src/types.ts +++ b/packages/openapi-parser/src/types.ts @@ -20,7 +20,7 @@ export interface OpenAPICustomSpecProperties { /** * Description in HTML format. */ - 'x-description-html'?: string; + 'x-gitbook-description-html'?: string; } /** @@ -41,7 +41,7 @@ export interface OpenAPICustomOperationProperties { /** * Description in HTML format. */ - 'x-description-html'?: string; + 'x-gitbook-description-html'?: string; } /** diff --git a/packages/react-openapi/src/resolveOpenAPIOperation.test.ts b/packages/react-openapi/src/resolveOpenAPIOperation.test.ts index 9ae6a3712..b4ad1222d 100644 --- a/packages/react-openapi/src/resolveOpenAPIOperation.test.ts +++ b/packages/react-openapi/src/resolveOpenAPIOperation.test.ts @@ -9,7 +9,7 @@ async function fetchFilesystem(url: string) { const filesystem = await parseOpenAPI({ value: text, rootURL: url }); const transformedFs = await traverse(filesystem, async (node) => { if ('description' in node && typeof node.description === 'string' && node.description) { - node['x-description-html'] = node.description; + node['x-gitbook-description-html'] = node.description; } return node; }); diff --git a/packages/react-openapi/src/utils.ts b/packages/react-openapi/src/utils.ts index e9ca30330..c3a45a578 100644 --- a/packages/react-openapi/src/utils.ts +++ b/packages/react-openapi/src/utils.ts @@ -12,8 +12,9 @@ export function createStateKey(key: string, scope?: string) { * Resolve the description of an object. */ export function resolveDescription(object: AnyObject) { - return 'x-description-html' in object && typeof object['x-description-html'] === 'string' - ? object['x-description-html'] + return 'x-gitbook-description-html' in object && + typeof object['x-gitbook-description-html'] === 'string' + ? object['x-gitbook-description-html'] : typeof object.description === 'string' ? object.description : undefined;