Files
Punsoong Khut 795d083264 feat(datasource): add APK datasource support (#40250)
* feat(datasource): add APK datasource support

* remove missing manager link for now

* lint docs

* update tests

* replace binary fixtures

* fix coverage

* Update lib/modules/datasource/apk/readme.md

Co-authored-by: Jamie Tanna <github@jamietanna.co.uk>

* Apply suggestion from @viceice

Co-authored-by: Michael Kriese <michael.kriese@gmx.de>

* use tar instead of tar-stream

* use codeBlock

* add ignore back

* make method private

* use an async extract instead of list

* add usage example

* stream apkindex download and parse line-by-line

* refactor(datasource): address review comments on APK datasource

Co-authored-by: Cursor <cursoragent@cursor.com>

* chore: sync pnpm-lock.yaml after merge

Co-authored-by: Cursor <cursoragent@cursor.com>

* test: remove ignored lines

As they seem to be covered, anyway.

* fixup! Merge remote-tracking branch 'origin/HEAD' into apk-datasource

* fixup! fixup! Merge remote-tracking branch 'origin/HEAD' into apk-datasource

* lint: use `node:` prefix

* Update lib/modules/datasource/apk/index.ts

Co-authored-by: Michael Kriese <michael.kriese@gmx.de>

* refactor(datasource/apk): use asTimestamp millis instead of new Date

Co-authored-by: Cursor <cursoragent@cursor.com>

* refactor(datasource/apk): rename uncached method to _getPackages

Co-authored-by: Cursor <cursoragent@cursor.com>

* refactor(datasource/apk): encode registry URL parts as query parameters

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(datasource/apk): reject invalid registry URLs, skip caching empty indexes

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(datasource/apk): throw on unparseable registry URL and drop unset sourceUrl support

Co-authored-by: Cursor <cursoragent@cursor.com>

* docs(datasource/apk): show how registry URL parameters build the index URLs

Co-authored-by: Cursor <cursoragent@cursor.com>

* Apply suggestions from code review

Co-authored-by: RahulGautamSingh <rahultesnik@gmail.com>

* fix(datasource/apk): keep credentials and port in component URLs

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(datasource/apk): extract APKINDEX by basename and key packages by name

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(datasource/apk): report a version served by two components only once

Co-authored-by: Cursor <cursoragent@cursor.com>

---------

Co-authored-by: Jamie Tanna <github@jamietanna.co.uk>
Co-authored-by: Michael Kriese <michael.kriese@gmx.de>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Jamie Tanna <jamie.tanna@mend.io>
Co-authored-by: RahulGautamSingh <rahultesnik@gmail.com>
2026-09-04 12:25:21 +00:00

206 lines
5.4 KiB
TypeScript

import { codeBlock } from 'common-tags';
import type { DirectoryResult } from 'tmp-promise';
import { dir } from 'tmp-promise';
import { GlobalConfig } from '../../../config/global.ts';
import * as memFs from '../../../util/fs/index.ts';
import { parseApkIndexFile } from './parser.ts';
describe('modules/datasource/apk/parser', () => {
describe('parseApkIndexFile', () => {
let cacheDir: DirectoryResult | null;
beforeEach(async () => {
cacheDir = await dir({ unsafeCleanup: true });
GlobalConfig.set({ cacheDir: cacheDir.path });
});
afterEach(async () => {
await cacheDir?.cleanup();
cacheDir = null;
});
it('should parse valid APK index content', async () => {
const indexContent = codeBlock`
P:bash
V:5.2.15-r0
U:https://alpinelinux.org/packages/bash
t:1700000000
P:nginx
V:1.24.0-r0
U:https://alpinelinux.org/packages/nginx
t:1700000001
`;
await memFs.outputCacheFile('APKINDEX', indexContent);
const result = await parseApkIndexFile('APKINDEX');
expect(result).toHaveLength(2);
expect(result[0]).toEqual({
name: 'bash',
version: '5.2.15-r0',
url: 'https://alpinelinux.org/packages/bash',
buildDate: 1700000000,
});
expect(result[1]).toEqual({
name: 'nginx',
version: '1.24.0-r0',
url: 'https://alpinelinux.org/packages/nginx',
buildDate: 1700000001,
});
});
it('should handle lines without colons', async () => {
const indexContent = codeBlock`
P:bash
V:5.2.15-r0
U:https://alpinelinux.org/packages/bash
t:1700000000
This line has no colon
Another line without colon
P:nginx
V:1.24.0-r0
`;
await memFs.outputCacheFile('APKINDEX', indexContent);
const result = await parseApkIndexFile('APKINDEX');
expect(result).toHaveLength(2);
expect(result[0].name).toBe('bash');
expect(result[1].name).toBe('nginx');
});
it('should skip packages missing required fields', async () => {
const indexContent = codeBlock`
P:bash
V:5.2.15-r0
U:https://alpinelinux.org/packages/bash
t:1700000000
P:incomplete-package
U:https://alpinelinux.org/packages/incomplete
t:1700000000
P:another-incomplete
V:1.0.0
P:nginx
V:1.24.0-r0
U:https://alpinelinux.org/packages/nginx
t:1700000001
`;
await memFs.outputCacheFile('APKINDEX', indexContent);
const result = await parseApkIndexFile('APKINDEX');
expect(result).toHaveLength(3);
expect(result[0].name).toBe('bash');
expect(result[1].name).toBe('another-incomplete');
expect(result[2].name).toBe('nginx');
});
it('should handle empty content', async () => {
await memFs.outputCacheFile('APKINDEX', '');
const result = await parseApkIndexFile('APKINDEX');
expect(result).toEqual([]);
});
it('should handle content with only whitespace', async () => {
await memFs.outputCacheFile('APKINDEX', ' \n\n \t \n ');
const result = await parseApkIndexFile('APKINDEX');
expect(result).toEqual([]);
});
it('should handle packages with only name and version', async () => {
const indexContent = codeBlock`
P:minimal-package
V:1.0.0
P:another-minimal
V:2.0.0
`;
await memFs.outputCacheFile('APKINDEX', indexContent);
const result = await parseApkIndexFile('APKINDEX');
expect(result).toHaveLength(2);
expect(result[0]).toEqual({
name: 'minimal-package',
version: '1.0.0',
});
expect(result[1]).toEqual({
name: 'another-minimal',
version: '2.0.0',
});
});
it('should handle packages with different field orders', async () => {
const indexContent = codeBlock`
t:1700000000
U:https://alpinelinux.org/packages/bash
V:5.2.15-r0
P:bash
P:nginx
t:1700000001
V:1.24.0-r0
U:https://alpinelinux.org/packages/nginx
`;
await memFs.outputCacheFile('APKINDEX', indexContent);
const result = await parseApkIndexFile('APKINDEX');
expect(result).toHaveLength(2);
expect(result[0]).toEqual({
name: 'bash',
version: '5.2.15-r0',
url: 'https://alpinelinux.org/packages/bash',
buildDate: 1700000000,
});
expect(result[1]).toEqual({
name: 'nginx',
version: '1.24.0-r0',
url: 'https://alpinelinux.org/packages/nginx',
buildDate: 1700000001,
});
});
it('should not push when a blank line ends an incomplete package', async () => {
const indexContent = codeBlock`
P:only-name
U:https://example.test/pkg
P:ok
V:1.0.0
`;
await memFs.outputCacheFile('APKINDEX', indexContent);
const result = await parseApkIndexFile('APKINDEX');
expect(result).toEqual([{ name: 'ok', version: '1.0.0' }]);
});
it('should not push trailing empty record when file ends with extra blank lines', async () => {
const indexContent = 'P:a\nV:1\n\n\n';
await memFs.outputCacheFile('APKINDEX', indexContent);
const result = await parseApkIndexFile('APKINDEX');
expect(result).toEqual([{ name: 'a', version: '1' }]);
});
});
});