feat(cache): Repository cache fingerprinting (#17367)

This commit is contained in:
Sergei Zharinov
2022-08-30 07:58:57 +03:00
committed by GitHub
parent 04ee279dcf
commit 6a93997863
15 changed files with 277 additions and 81 deletions
+23 -4
View File
@@ -3,10 +3,11 @@ import type {
RepoCacheRecordV10,
RepoCacheRecordV11,
RepoCacheRecordV12,
RepoCacheRecordV13,
} from './types';
// Increment this whenever there could be incompatibilities between old and new cache structure
export const CACHE_REVISION = 12;
export const CACHE_REVISION = 13;
export function isValidRev10(
input: unknown,
@@ -35,17 +36,35 @@ export function isValidRev11(
);
}
export function isValidRev12(
function isValidRev12Shape(
input: unknown,
repo?: string
): input is RepoCacheRecordV12 {
return (
is.plainObject(input) &&
is.safeInteger(input.revision) &&
input.revision === CACHE_REVISION &&
is.string(input.repository) &&
(!repo || repo === input.repository) &&
is.string(input.payload) &&
is.string(input.hash)
);
}
export function isValidRev12(
input: unknown,
repo?: string
): input is RepoCacheRecordV12 {
return (
isValidRev12Shape(input, repo) &&
input.revision === 12 &&
(!repo || repo === input.repository)
);
}
export function isValidRev13(input: unknown): input is RepoCacheRecordV13 {
return (
is.plainObject(input) &&
is.string(input.fingerprint) &&
isValidRev12Shape(input) &&
input.revision === 13
);
}
+50 -20
View File
@@ -9,6 +9,7 @@ import {
isValidRev10,
isValidRev11,
isValidRev12,
isValidRev13,
} from '../common';
import type {
RepoCache,
@@ -17,6 +18,7 @@ import type {
RepoCacheRecordV10,
RepoCacheRecordV11,
RepoCacheRecordV12,
RepoCacheRecordV13,
} from '../types';
const compress = promisify(zlib.brotliCompress);
@@ -27,12 +29,25 @@ export abstract class RepoCacheBase implements RepoCache {
private oldHash: string | null = null;
private data: RepoCacheData = {};
protected constructor(protected readonly repository: string) {}
protected constructor(
protected readonly repository: string,
protected readonly fingerprint: string
) {}
protected abstract read(): Promise<string | null>;
protected abstract write(data: RepoCacheRecord): Promise<void>;
private restoreFromRev10(oldCache: RepoCacheRecordV10): void {
delete oldCache.repository;
delete oldCache.revision;
this.data = oldCache;
}
private restoreFromRev11(oldCache: RepoCacheRecordV11): void {
this.data = oldCache.data;
}
private async restoreFromRev12(oldCache: RepoCacheRecordV12): Promise<void> {
const compressed = Buffer.from(oldCache.payload, 'base64');
const uncompressed = await decompress(compressed);
@@ -41,26 +56,30 @@ export abstract class RepoCacheBase implements RepoCache {
this.oldHash = oldCache.hash;
}
private restoreFromRev11(oldCache: RepoCacheRecordV11): void {
this.data = oldCache.data;
}
private restoreFromRev10(oldCache: RepoCacheRecordV10): void {
delete oldCache.repository;
delete oldCache.revision;
this.data = oldCache;
private async restoreFromRev13(oldCache: RepoCacheRecordV13): Promise<void> {
if (oldCache.fingerprint !== this.fingerprint) {
logger.debug('Repository cache fingerprint is invalid');
return;
}
await this.restoreFromRev12(oldCache);
}
async load(): Promise<void> {
try {
const data = await this.read();
if (!is.string(data)) {
const rawOldCache = await this.read();
if (!is.string(rawOldCache)) {
logger.debug(
`RepoCacheBase.load() - expecting data of type 'string' received '${typeof data}' instead - skipping`
`RepoCacheBase.load() - expecting data of type 'string' received '${typeof rawOldCache}' instead - skipping`
);
return;
}
const oldCache = JSON.parse(data) as unknown;
const oldCache = JSON.parse(rawOldCache) as unknown;
if (isValidRev13(oldCache)) {
await this.restoreFromRev13(oldCache);
logger.debug('Repository cache is restored from revision 13');
return;
}
if (isValidRev12(oldCache, this.repository)) {
await this.restoreFromRev12(oldCache);
@@ -87,15 +106,26 @@ export abstract class RepoCacheBase implements RepoCache {
}
async save(): Promise<void> {
const jsonStr = JSON.stringify(this.data);
const hash = await hasha.async(jsonStr);
if (hash === this.oldHash) {
return;
}
const revision = CACHE_REVISION;
const repository = this.repository;
const jsonStr = JSON.stringify(this.data);
const hash = await hasha.async(jsonStr, { algorithm: 'sha256' });
if (hash !== this.oldHash) {
const compressed = await compress(jsonStr);
const payload = compressed.toString('base64');
await this.write({ revision, repository, payload, hash });
}
const fingerprint = this.fingerprint;
const compressedPayload = await compress(jsonStr);
const payload = compressedPayload.toString('base64');
await this.write({
revision,
repository,
fingerprint,
payload,
hash,
});
}
getData(): RepoCacheData {
+5 -4
View File
@@ -7,20 +7,21 @@ import { RepoCacheS3 } from './s3';
export class CacheFactory {
static get(
repository: string,
cacheType: RepositoryCacheType = 'local'
repoFingerprint: string,
cacheType: RepositoryCacheType
): RepoCache {
const type = cacheType.split('://')[0].trim().toLowerCase();
switch (type) {
case 'local':
return new RepoCacheLocal(repository);
return new RepoCacheLocal(repository, repoFingerprint);
case 's3':
return new RepoCacheS3(repository, cacheType);
return new RepoCacheS3(repository, repoFingerprint, cacheType);
default:
logger.warn(
{ cacheType },
`Repository cache type not supported using type "local" instead`
);
return new RepoCacheLocal(repository);
return new RepoCacheLocal(repository, repoFingerprint);
}
}
}
+147 -29
View File
@@ -18,26 +18,45 @@ async function createCacheRecord(
repository = 'some/repo'
): Promise<RepoCacheRecord> {
const revision = CACHE_REVISION;
const fingerprint = '0123456789abcdef';
const jsonStr = JSON.stringify(data);
const hash = hasha(jsonStr, { algorithm: 'sha256' });
const compressed = await compress(jsonStr);
const payload = compressed.toString('base64');
return { revision, repository, payload, hash };
const hash = hasha(jsonStr);
const compressedPayload = await compress(jsonStr);
const payload = compressedPayload.toString('base64');
return {
revision,
repository,
fingerprint,
payload,
hash,
};
}
describe('util/cache/repository/impl/local', () => {
beforeEach(() => {
jest.resetAllMocks();
GlobalConfig.set({ cacheDir: '/tmp/cache', platform: 'github' });
fs.cachePathExists.mockResolvedValue(true);
});
it('returns empty object before any data load', () => {
const localRepoCache = CacheFactory.get('some/repo', 'local');
const localRepoCache = CacheFactory.get(
'some/repo',
'0123456789abcdef',
'local'
);
expect(localRepoCache.getData()).toBeEmpty();
});
it('skip when receives non-string data', async () => {
const localRepoCache = CacheFactory.get('some/repo', 'local');
fs.cachePathExists.mockResolvedValueOnce(true);
const localRepoCache = CacheFactory.get(
'some/repo',
'0123456789abcdef',
'local'
);
await localRepoCache.load(); // readCacheFile is mocked but has no return value set - therefore returns undefined
expect(logger.debug).toHaveBeenCalledWith(
"RepoCacheBase.load() - expecting data of type 'string' received 'undefined' instead - skipping"
@@ -45,7 +64,11 @@ describe('util/cache/repository/impl/local', () => {
});
it('skip when not found', async () => {
const localRepoCache = CacheFactory.get('some/repo', 'local');
const localRepoCache = CacheFactory.get(
'some/repo',
'0123456789abcdef',
'local'
);
await localRepoCache.load(); // readCacheFile is mocked but has no return value set - therefore returns undefined
expect(logger.debug).not.toHaveBeenCalledWith();
});
@@ -53,17 +76,36 @@ describe('util/cache/repository/impl/local', () => {
it('loads previously stored cache from disk', async () => {
const data: RepoCacheData = { semanticCommits: 'enabled' };
const cacheRecord = await createCacheRecord(data);
fs.cachePathExists.mockResolvedValueOnce(true);
fs.readCacheFile.mockResolvedValue(JSON.stringify(cacheRecord));
const localRepoCache = CacheFactory.get('some/repo', 'local');
const localRepoCache = CacheFactory.get(
'some/repo',
'0123456789abcdef',
'local'
);
await localRepoCache.load();
expect(localRepoCache.getData()).toEqual(data);
});
it('migrates revision from 10 to 12', async () => {
fs.cachePathExists.mockResolvedValueOnce(true);
it('resets if fingerprint does not match', async () => {
const data: RepoCacheData = { semanticCommits: 'enabled' };
const cacheRecord: RepoCacheRecord = {
...(await createCacheRecord(data)),
fingerprint: '111',
};
fs.readCacheFile.mockResolvedValue(JSON.stringify(cacheRecord));
const cache1 = CacheFactory.get('some/repo', '111', 'local');
await cache1.load();
expect(cache1.getData()).toEqual(data);
const cache2 = CacheFactory.get('some/repo', '222', 'local');
await cache2.load();
expect(cache2.getData()).toBeEmpty();
});
it('migrates revision from 10 to 13', async () => {
fs.readCacheFile.mockResolvedValue(
JSON.stringify({
revision: 10,
@@ -71,7 +113,11 @@ describe('util/cache/repository/impl/local', () => {
semanticCommits: 'enabled',
})
);
const localRepoCache = CacheFactory.get('some/repo', 'local');
const localRepoCache = CacheFactory.get(
'some/repo',
'0123456789abcdef',
'local'
);
await localRepoCache.load();
await localRepoCache.save();
@@ -83,8 +129,7 @@ describe('util/cache/repository/impl/local', () => {
);
});
it('migrates revision from 11 to 12', async () => {
fs.cachePathExists.mockResolvedValueOnce(true);
it('migrates revision from 11 to 13', async () => {
fs.readCacheFile.mockResolvedValue(
JSON.stringify({
revision: 11,
@@ -92,7 +137,11 @@ describe('util/cache/repository/impl/local', () => {
data: { semanticCommits: 'enabled' },
})
);
const localRepoCache = CacheFactory.get('some/repo', 'local');
const localRepoCache = CacheFactory.get(
'some/repo',
'0123456789abcdef',
'local'
);
await localRepoCache.load();
await localRepoCache.save();
@@ -104,8 +153,36 @@ describe('util/cache/repository/impl/local', () => {
);
});
it('migrates revision from 12 to 13', async () => {
const { repository, payload, hash } = await createCacheRecord({
semanticCommits: 'enabled',
});
fs.readCacheFile.mockResolvedValue(
JSON.stringify({ revision: 12, repository, payload, hash })
);
const localRepoCache = CacheFactory.get(
'some/repo',
'0123456789abcdef',
'local'
);
await localRepoCache.load();
const data = localRepoCache.getData();
data.semanticCommits = 'disabled';
await localRepoCache.save();
expect(fs.outputCacheFile).toHaveBeenCalledWith(
'/tmp/cache/renovate/repository/github/some/repo.json',
JSON.stringify(
await createCacheRecord({
semanticCommits: 'disabled',
})
)
);
});
it('does not migrate from older revisions to 11', async () => {
fs.cachePathExists.mockResolvedValueOnce(true);
fs.readCacheFile.mockResolvedValueOnce(
JSON.stringify({
revision: 9,
@@ -114,16 +191,23 @@ describe('util/cache/repository/impl/local', () => {
})
);
const localRepoCache = CacheFactory.get('some/repo', 'local');
const localRepoCache = CacheFactory.get(
'some/repo',
'0123456789abcdef',
'local'
);
await localRepoCache.load();
expect(localRepoCache.getData()).toBeEmpty();
});
it('handles invalid data', async () => {
fs.cachePathExists.mockResolvedValueOnce(true);
fs.readCacheFile.mockResolvedValue(JSON.stringify({ foo: 'bar' }));
const localRepoCache = CacheFactory.get('some/repo', 'local');
const localRepoCache = CacheFactory.get(
'some/repo',
'0123456789abcdef',
'local'
);
await localRepoCache.load();
@@ -131,9 +215,12 @@ describe('util/cache/repository/impl/local', () => {
});
it('handles file read error', async () => {
fs.cachePathExists.mockResolvedValueOnce(true);
fs.readCacheFile.mockRejectedValue(new Error('unknown error'));
const localRepoCache = CacheFactory.get('some/repo', 'local');
const localRepoCache = CacheFactory.get(
'some/repo',
'0123456789abcdef',
'local'
);
await localRepoCache.load();
@@ -142,9 +229,12 @@ describe('util/cache/repository/impl/local', () => {
});
it('handles invalid json', async () => {
fs.cachePathExists.mockResolvedValueOnce(true);
fs.readCacheFile.mockResolvedValue('{1');
const localRepoCache = CacheFactory.get('some/repo', 'local');
const localRepoCache = CacheFactory.get(
'some/repo',
'0123456789abcdef',
'local'
);
await localRepoCache.load();
@@ -153,21 +243,29 @@ describe('util/cache/repository/impl/local', () => {
it('resets if repository does not match', async () => {
const cacheRecord = createCacheRecord({ semanticCommits: 'enabled' });
fs.cachePathExists.mockResolvedValueOnce(true);
fs.readCacheFile.mockResolvedValueOnce(JSON.stringify(cacheRecord));
const localRepoCache = CacheFactory.get('some/repo', 'local');
const localRepoCache = CacheFactory.get(
'some/repo',
'0123456789abcdef',
'local'
);
await localRepoCache.load();
expect(localRepoCache.getData()).toBeEmpty();
});
it('saves modified cache data to file', async () => {
const oldCacheRecord = createCacheRecord({ semanticCommits: 'enabled' });
const oldCacheRecord = await createCacheRecord({
semanticCommits: 'enabled',
});
const cacheType = 'protocol://domain/path';
fs.cachePathExists.mockResolvedValueOnce(true);
fs.readCacheFile.mockResolvedValueOnce(JSON.stringify(oldCacheRecord));
const localRepoCache = CacheFactory.get('some/repo', cacheType);
const localRepoCache = CacheFactory.get(
'some/repo',
'0123456789abcdef',
cacheType
);
await localRepoCache.load();
const data = localRepoCache.getData();
data.semanticCommits = 'disabled';
@@ -186,4 +284,24 @@ describe('util/cache/repository/impl/local', () => {
JSON.stringify(newCacheRecord)
);
});
it('does not write cache that is not changed', async () => {
const oldCacheRecord = await createCacheRecord({
semanticCommits: 'enabled',
});
const cacheType = 'protocol://domain/path';
fs.readCacheFile.mockResolvedValueOnce(JSON.stringify(oldCacheRecord));
const localRepoCache = CacheFactory.get(
'some/repo',
'0123456789abcdef',
cacheType
);
await localRepoCache.load();
expect(localRepoCache.getData()).toEqual({ semanticCommits: 'enabled' });
await localRepoCache.save();
expect(fs.outputCacheFile).not.toHaveBeenCalledWith();
});
});
+2 -2
View File
@@ -6,8 +6,8 @@ import type { RepoCacheRecord } from '../types';
import { RepoCacheBase } from './base';
export class RepoCacheLocal extends RepoCacheBase {
constructor(repository: string) {
super(repository);
constructor(repository: string, fingerprint: string) {
super(repository, fingerprint);
}
protected async read(): Promise<string | null> {
+2 -2
View File
@@ -57,7 +57,7 @@ describe('util/cache/repository/impl/s3', () => {
GlobalConfig.set({ platform: 'github' });
jest.clearAllMocks();
s3Mock.reset();
s3Cache = new RepoCacheS3(repository, url);
s3Cache = new RepoCacheS3(repository, '0123456789abcdef', url);
getObjectCommandInput = createGetObjectCommandInput(repository, url);
putObjectCommandInput = createPutObjectCommandInput(
repository,
@@ -127,7 +127,7 @@ describe('util/cache/repository/impl/s3', () => {
});
it('creates an S3 client using the cache factory', () => {
const cache = CacheFactory.get(repository, url);
const cache = CacheFactory.get(repository, '0123456789abcdef', url);
expect(cache instanceof RepoCacheS3).toBeTrue();
});
});
+2 -2
View File
@@ -15,8 +15,8 @@ export class RepoCacheS3 extends RepoCacheBase {
private readonly s3Client;
private readonly bucket;
constructor(repository: string, url: string) {
super(repository);
constructor(repository: string, fingerprint: string, url: string) {
super(repository, fingerprint);
this.bucket = parseS3Url(url)?.Bucket;
this.s3Client = getS3Client();
}
+3 -3
View File
@@ -1,8 +1,8 @@
import { mocked } from '../../../../test/util';
import { GlobalConfig } from '../../../config/global';
import type { RenovateConfig } from '../../../config/types';
import * as _fs from '../../fs';
import { initRepoCache } from './init';
import type { RepoCacheConfig } from './types';
import { getCache, resetCache, saveCache } from '.';
jest.mock('../../fs');
@@ -16,10 +16,10 @@ describe('util/cache/repository/index', () => {
GlobalConfig.set({ cacheDir: '/tmp/cache', platform: 'github' });
});
const config: RenovateConfig = {
platform: 'github',
const config: RepoCacheConfig = {
repository: 'some/repo',
repositoryCache: 'enabled',
repoFingerprint: '0123456789abcdef',
};
it('returns if cache not enabled', async () => {
+10 -5
View File
@@ -1,15 +1,20 @@
import type { RenovateConfig } from '../../../config/types';
import { CacheFactory } from './impl/cache-factory';
import { RepoCacheNull } from './impl/null';
import type { RepoCacheConfig } from './types';
import { resetCache, setCache } from '.';
/**
* Extracted to separate file in order to avoid circular module dependencies.
*/
export async function initRepoCache(config: RenovateConfig): Promise<void> {
export async function initRepoCache(config: RepoCacheConfig): Promise<void> {
resetCache();
const { repository, repositoryCache, repositoryCacheType: type } = config;
const {
repository,
repositoryCache,
repositoryCacheType: type = 'local',
repoFingerprint,
} = config;
if (repositoryCache === 'disabled') {
setCache(new RepoCacheNull());
@@ -17,14 +22,14 @@ export async function initRepoCache(config: RenovateConfig): Promise<void> {
}
if (repositoryCache === 'enabled') {
const cache = CacheFactory.get(repository!, type);
const cache = CacheFactory.get(repository!, repoFingerprint, type);
await cache.load();
setCache(cache);
return;
}
if (repositoryCache === 'reset') {
const cache = CacheFactory.get(repository!, type);
const cache = CacheFactory.get(repository!, repoFingerprint, type);
await cache.save();
setCache(cache);
return;
+16 -1
View File
@@ -1,3 +1,7 @@
import type {
RepositoryCacheConfig,
RepositoryCacheType,
} from '../../../config/types';
import type { PackageFile } from '../../../modules/manager/types';
import type { RepoInitConfig } from '../../../workers/repository/init/types';
import type { GitConflictsCache } from '../../git/types';
@@ -65,10 +69,21 @@ export interface RepoCacheRecordV12 {
hash: string;
}
export type RepoCacheRecord = RepoCacheRecordV12;
export interface RepoCacheRecordV13 extends RepoCacheRecordV12 {
fingerprint: string;
}
export type RepoCacheRecord = RepoCacheRecordV13;
export interface RepoCache {
load(): Promise<void>;
save(): Promise<void>;
getData(): RepoCacheData;
}
export interface RepoCacheConfig {
repository?: string;
repositoryCache?: RepositoryCacheConfig;
repositoryCacheType?: RepositoryCacheType;
repoFingerprint: string;
}
+9 -3
View File
@@ -1,13 +1,19 @@
import { RenovateConfig, getConfig } from '../../../../test/util';
import { getConfig } from '../../../../test/util';
import { GlobalConfig } from '../../../config/global';
import type { WorkerPlatformConfig } from './apis';
import { initializeCaches } from './cache';
describe('workers/repository/init/cache', () => {
describe('initializeCaches()', () => {
let config: RenovateConfig;
let config: WorkerPlatformConfig;
beforeEach(() => {
config = { ...getConfig() };
config = {
...getConfig(),
repoFingerprint: '0123456789abcdef',
defaultBranch: 'main',
isFork: false,
};
GlobalConfig.set({ cacheDir: '' });
});
+4 -2
View File
@@ -1,10 +1,10 @@
import fs from 'fs-extra';
import type { RenovateConfig } from '../../../config/types';
import * as npmApi from '../../../modules/datasource/npm';
import * as memCache from '../../../util/cache/memory';
import * as repositoryCache from '../../../util/cache/repository';
import { initRepoCache } from '../../../util/cache/repository/init';
import { privateCacheDir } from '../../../util/fs';
import type { WorkerPlatformConfig } from './apis';
export async function resetCaches(): Promise<void> {
memCache.reset();
@@ -13,7 +13,9 @@ export async function resetCaches(): Promise<void> {
npmApi.resetMemCache();
}
export async function initializeCaches(config: RenovateConfig): Promise<void> {
export async function initializeCaches(
config: WorkerPlatformConfig
): Promise<void> {
memCache.init();
await initRepoCache(config);
await fs.ensureDir(privateCacheDir());
+2 -2
View File
@@ -8,7 +8,7 @@ import { setUserRepoConfig } from '../../../util/git';
import { getAll } from '../../../util/host-rules';
import { checkIfConfigured } from '../configured';
import { PackageFiles } from '../package-files';
import { initApis } from './apis';
import { WorkerPlatformConfig, initApis } from './apis';
import { initializeCaches, resetCaches } from './cache';
import { getRepoConfig } from './config';
import { detectVulnerabilityAlerts } from './vulnerability';
@@ -34,7 +34,7 @@ export async function initRepo(
let config: RenovateConfig = initializeConfig(config_);
await resetCaches();
config = await initApis(config);
await initializeCaches(config);
await initializeCaches(config as WorkerPlatformConfig);
config = await getRepoConfig(config);
checkIfConfigured(config);
warnOnUnsupportedOptions(config);
+1 -1
View File
@@ -36,7 +36,7 @@ jest.mock('../../../config/migrate-validate');
describe('workers/repository/init/merge', () => {
describe('detectRepoFileConfig()', () => {
beforeEach(async () => {
await initRepoCache({});
await initRepoCache({ repoFingerprint: '0123456789abcdef' });
});
it('returns config if not found', async () => {
+1 -1
View File
@@ -16,7 +16,7 @@ beforeEach(() => {
describe('workers/repository/init/semantic', () => {
describe('detectSemanticCommits()', () => {
beforeEach(async () => {
await initRepoCache({});
await initRepoCache({ repoFingerprint: '0123456789abcdef' });
});
it('detects false if unknown', async () => {