From 6ae3818aabe7d14cac0342c513859c685b023383 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20B=C3=B6sel?= Date: Mon, 29 Apr 2024 10:03:34 +0200 Subject: [PATCH] feat(manager/pipenv): Support custom environment variable usage in Pipfile source URLs (#28062) Co-authored-by: Michael Kriese Co-authored-by: Rhys Arkins --- lib/modules/manager/pipenv/artifacts.spec.ts | 40 +++++- lib/modules/manager/pipenv/artifacts.ts | 131 +++++++++++++------ 2 files changed, 130 insertions(+), 41 deletions(-) diff --git a/lib/modules/manager/pipenv/artifacts.spec.ts b/lib/modules/manager/pipenv/artifacts.spec.ts index 0027539a09332d..6d6d6bf8f3f332 100644 --- a/lib/modules/manager/pipenv/artifacts.spec.ts +++ b/lib/modules/manager/pipenv/artifacts.spec.ts @@ -12,11 +12,18 @@ import { } from '../../../../test/util'; import { GlobalConfig } from '../../../config/global'; import type { RepoGlobalConfig } from '../../../config/types'; +import { logger } from '../../../logger'; import * as docker from '../../../util/exec/docker'; +import type { ExtraEnv, Opt } from '../../../util/exec/types'; import type { StatusResult } from '../../../util/git/types'; import { find as _find } from '../../../util/host-rules'; import * as _datasource from '../../datasource'; import type { UpdateArtifactsConfig } from '../types'; +import { + addExtraEnvVariable, + extractEnvironmentVariableName, + getMatchingHostRule, +} from './artifacts'; import type { PipfileLockSchema } from './schema'; import { updateArtifacts } from '.'; @@ -626,7 +633,31 @@ describe('modules/manager/pipenv/artifacts', () => { ]); }); - it('does not pass private credential environment vars if variable names differ from allowed', async () => { + it('returns no host rule on invalid url', () => { + expect(getMatchingHostRule('')).toBeNull(); + }); + + it.each` + credential | result + ${'$USERNAME'} | ${'USERNAME'} + ${'$'} | ${null} + ${''} | ${null} + ${'${USERNAME}'} | ${'USERNAME'} + ${'${USERNAME:-default}'} | ${'USERNAME'} + ${'${COMPLEX_NAME_1:-default}'} | ${'COMPLEX_NAME_1'} + `('extractEnvironmentVariableName(%p)', ({ credential, result }) => { + expect(extractEnvironmentVariableName(credential)).toEqual(result); + }); + + it('warns about duplicate placeholders with different values', () => { + const extraEnv: Opt = { + FOO: '1', + }; + addExtraEnvVariable(extraEnv, 'FOO', '2'); + expect(logger.warn).toHaveBeenCalledOnce(); + }); + + it('updates extraEnv if variable names differ from default', async () => { fs.ensureCacheDir.mockResolvedValueOnce(pipenvCacheDir); fs.ensureCacheDir.mockResolvedValueOnce(pipCacheDir); fs.ensureCacheDir.mockResolvedValueOnce(virtualenvsCacheDir); @@ -639,6 +670,11 @@ describe('modules/manager/pipenv/artifacts', () => { ); fs.readLocalFile.mockResolvedValueOnce('New Pipfile.lock'); + find.mockReturnValueOnce({ + username: 'usernameOne', + password: 'passwordTwo', + }); + expect( await updateArtifacts({ packageFileName: 'Pipfile', @@ -664,6 +700,8 @@ describe('modules/manager/pipenv/artifacts', () => { env: { PIPENV_CACHE_DIR: pipenvCacheDir, WORKON_HOME: virtualenvsCacheDir, + USERNAME_FOO: 'usernameOne', + PAZZWORD: 'passwordTwo', }, }, }, diff --git a/lib/modules/manager/pipenv/artifacts.ts b/lib/modules/manager/pipenv/artifacts.ts index 86a517ce7d5657..5c0e98f5550e7b 100644 --- a/lib/modules/manager/pipenv/artifacts.ts +++ b/lib/modules/manager/pipenv/artifacts.ts @@ -13,6 +13,8 @@ import { } from '../../../util/fs'; import { getRepoStatus } from '../../../util/git'; import { find } from '../../../util/host-rules'; +import { regEx } from '../../../util/regex'; +import { parseUrl } from '../../../util/url'; import { PypiDatasource } from '../../datasource/pypi'; import type { UpdateArtifact, @@ -113,35 +115,103 @@ export function getPipenvConstraint( return ''; } -function getMatchingHostRule(url: string): HostRule { - return find({ hostType: PypiDatasource.id, url }); +export function getMatchingHostRule(url: string): HostRule | null { + const parsedUrl = parseUrl(url); + if (parsedUrl) { + parsedUrl.username = ''; + parsedUrl.password = ''; + const urlWithoutCredentials = parsedUrl.toString(); + + return find({ hostType: PypiDatasource.id, url: urlWithoutCredentials }); + } + return null; } -async function findPipfileSourceUrlWithCredentials( +async function findPipfileSourceUrlsWithCredentials( pipfileContent: string, pipfileName: string, -): Promise { +): Promise { const pipfile = await extractPackageFile(pipfileContent, pipfileName); - if (!pipfile) { - logger.debug('Error parsing Pipfile'); - return null; - } - const credentialTokens = [ - '$USERNAME:', - // eslint-disable-next-line no-template-curly-in-string - '${USERNAME}', - '$PASSWORD@', - // eslint-disable-next-line no-template-curly-in-string - '${PASSWORD}', - ]; + return ( + pipfile?.registryUrls + ?.map(parseUrl) + .filter(is.urlInstance) + .filter((url) => is.nonEmptyStringAndNotWhitespace(url.username)) ?? [] + ); +} - const sourceWithCredentials = pipfile.registryUrls?.find((url) => - credentialTokens.some((token) => url.includes(token)), +/** + * This will extract the actual variable name from an environment-placeholder: + * ${USERNAME:-defaultvalue} will yield 'USERNAME' + */ +export function extractEnvironmentVariableName( + credential: string, +): string | null { + const match = regEx('([a-z0-9_]+)', 'i').exec(decodeURI(credential)); + return match?.length ? match[0] : null; +} + +export function addExtraEnvVariable( + extraEnv: ExtraEnv, + environmentVariableName: string, + environmentValue: string, +): void { + logger.trace( + `Adding ${environmentVariableName} environment variable for pipenv`, ); + if ( + extraEnv[environmentVariableName] && + extraEnv[environmentVariableName] !== environmentValue + ) { + logger.warn( + `Possible misconfiguration, ${environmentVariableName} is already set to a different value`, + ); + } + extraEnv[environmentVariableName] = environmentValue; +} - // Only one source is currently supported - return sourceWithCredentials ?? null; +/** + * Pipenv allows configuring source-urls for remote repositories with placeholders for credentials, i.e. http://$USER:$PASS@myprivate.repo + * if a matching host rule exists for that repository, we need to set the corresponding variables. + * Simply substituting them in the URL is not an option as it would impact the hash for the resulting Pipfile.lock + * + */ +async function addCredentialsForSourceUrls( + newPipfileContent: string, + pipfileName: string, + extraEnv: ExtraEnv, +): Promise { + const sourceUrls = await findPipfileSourceUrlsWithCredentials( + newPipfileContent, + pipfileName, + ); + for (const parsedSourceUrl of sourceUrls) { + logger.trace(`Trying to add credentials for ${parsedSourceUrl.toString()}`); + const matchingHostRule = getMatchingHostRule(parsedSourceUrl.toString()); + if (matchingHostRule) { + const usernameVariableName = extractEnvironmentVariableName( + parsedSourceUrl.username, + ); + if (matchingHostRule.username && usernameVariableName) { + addExtraEnvVariable( + extraEnv, + usernameVariableName, + matchingHostRule.username, + ); + } + const passwordVariableName = extractEnvironmentVariableName( + parsedSourceUrl.password, + ); + if (matchingHostRule.password && passwordVariableName) { + addExtraEnvVariable( + extraEnv, + passwordVariableName, + matchingHostRule.password, + ); + } + } + } } export async function updateArtifacts({ @@ -188,26 +258,7 @@ export async function updateArtifacts({ }, ], }; - - const sourceUrl = await findPipfileSourceUrlWithCredentials( - newPipfileContent, - pipfileName, - ); - if (sourceUrl) { - logger.debug({ sourceUrl }, 'Pipfile contains credentials'); - const hostRule = getMatchingHostRule(sourceUrl); - if (hostRule) { - logger.debug('Found matching hostRule for Pipfile credentials'); - if (hostRule.username) { - logger.debug('Adding USERNAME environment variable for pipenv'); - extraEnv.USERNAME = hostRule.username; - } - if (hostRule.password) { - logger.debug('Adding PASSWORD environment variable for pipenv'); - extraEnv.PASSWORD = hostRule.password; - } - } - } + await addCredentialsForSourceUrls(newPipfileContent, pipfileName, extraEnv); execOptions.extraEnv = extraEnv; logger.trace({ cmd }, 'pipenv lock command');