Skip to content

Commit

Permalink
feat(manager/pipenv): Support custom environment variable usage in Pi…
Browse files Browse the repository at this point in the history 
…pfile source URLs (#28062)

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
Co-authored-by: Rhys Arkins <rhys@arkins.net>
  • Loading branch information
@fivetide @viceice @rarkins
3 people committed Apr 29, 2024
1 parent 0962edf commit 6ae3818
Show file tree
Hide file tree
Showing 2 changed files with 130 additions and 41 deletions.
40 changes: 39 additions & 1 deletion lib/modules/manager/pipenv/artifacts.spec.ts
View file
Expand Up @@ -12,11 +12,18 @@ import {
} from '../../../../test/util';
import { GlobalConfig } from '../../../config/global';
import type { RepoGlobalConfig } from '../../../config/types';
import { logger } from '../../../logger';
import * as docker from '../../../util/exec/docker';
import type { ExtraEnv, Opt } from '../../../util/exec/types';
import type { StatusResult } from '../../../util/git/types';
import { find as _find } from '../../../util/host-rules';
import * as _datasource from '../../datasource';
import type { UpdateArtifactsConfig } from '../types';
import {
addExtraEnvVariable,
extractEnvironmentVariableName,
getMatchingHostRule,
} from './artifacts';
import type { PipfileLockSchema } from './schema';
import { updateArtifacts } from '.';

Expand Down Expand Up @@ -626,7 +633,31 @@ describe('modules/manager/pipenv/artifacts', () => {
]);
});

it('does not pass private credential environment vars if variable names differ from allowed', async () => {
it('returns no host rule on invalid url', () => {
expect(getMatchingHostRule('')).toBeNull();
});

it.each`
credential | result
${'$USERNAME'} | ${'USERNAME'}
${'$'} | ${null}
${''} | ${null}
${'${USERNAME}'} | ${'USERNAME'}
${'${USERNAME:-default}'} | ${'USERNAME'}
${'${COMPLEX_NAME_1:-default}'} | ${'COMPLEX_NAME_1'}
`('extractEnvironmentVariableName(%p)', ({ credential, result }) => {
expect(extractEnvironmentVariableName(credential)).toEqual(result);
});

it('warns about duplicate placeholders with different values', () => {
const extraEnv: Opt<ExtraEnv> = {
FOO: '1',
};
addExtraEnvVariable(extraEnv, 'FOO', '2');
expect(logger.warn).toHaveBeenCalledOnce();
});

it('updates extraEnv if variable names differ from default', async () => {
fs.ensureCacheDir.mockResolvedValueOnce(pipenvCacheDir);
fs.ensureCacheDir.mockResolvedValueOnce(pipCacheDir);
fs.ensureCacheDir.mockResolvedValueOnce(virtualenvsCacheDir);
Expand All @@ -639,6 +670,11 @@ describe('modules/manager/pipenv/artifacts', () => {
);
fs.readLocalFile.mockResolvedValueOnce('New Pipfile.lock');

find.mockReturnValueOnce({
username: 'usernameOne',
password: 'passwordTwo',
});

expect(
await updateArtifacts({
packageFileName: 'Pipfile',
Expand All @@ -664,6 +700,8 @@ describe('modules/manager/pipenv/artifacts', () => {
env: {
PIPENV_CACHE_DIR: pipenvCacheDir,
WORKON_HOME: virtualenvsCacheDir,
USERNAME_FOO: 'usernameOne',
PAZZWORD: 'passwordTwo',
},
},
},
Expand Down
131 changes: 91 additions & 40 deletions lib/modules/manager/pipenv/artifacts.ts
View file
Expand Up @@ -13,6 +13,8 @@ import {
} from '../../../util/fs';
import { getRepoStatus } from '../../../util/git';
import { find } from '../../../util/host-rules';
import { regEx } from '../../../util/regex';
import { parseUrl } from '../../../util/url';
import { PypiDatasource } from '../../datasource/pypi';
import type {
UpdateArtifact,
Expand Down Expand Up @@ -113,35 +115,103 @@ export function getPipenvConstraint(
return '';
}

function getMatchingHostRule(url: string): HostRule {
return find({ hostType: PypiDatasource.id, url });
export function getMatchingHostRule(url: string): HostRule | null {
const parsedUrl = parseUrl(url);
if (parsedUrl) {
parsedUrl.username = '';
parsedUrl.password = '';
const urlWithoutCredentials = parsedUrl.toString();

return find({ hostType: PypiDatasource.id, url: urlWithoutCredentials });
}
return null;
}

async function findPipfileSourceUrlWithCredentials(
async function findPipfileSourceUrlsWithCredentials(
pipfileContent: string,
pipfileName: string,
): Promise<string | null> {
): Promise<URL[]> {
const pipfile = await extractPackageFile(pipfileContent, pipfileName);
if (!pipfile) {
logger.debug('Error parsing Pipfile');
return null;
}

const credentialTokens = [
'$USERNAME:',
// eslint-disable-next-line no-template-curly-in-string
'${USERNAME}',
'$PASSWORD@',
// eslint-disable-next-line no-template-curly-in-string
'${PASSWORD}',
];
return (
pipfile?.registryUrls
?.map(parseUrl)
.filter(is.urlInstance)
.filter((url) => is.nonEmptyStringAndNotWhitespace(url.username)) ?? []
);
}

const sourceWithCredentials = pipfile.registryUrls?.find((url) =>
credentialTokens.some((token) => url.includes(token)),
/**
* This will extract the actual variable name from an environment-placeholder:
* ${USERNAME:-defaultvalue} will yield 'USERNAME'
*/
export function extractEnvironmentVariableName(
credential: string,
): string | null {
const match = regEx('([a-z0-9_]+)', 'i').exec(decodeURI(credential));
return match?.length ? match[0] : null;
}

export function addExtraEnvVariable(
extraEnv: ExtraEnv<unknown>,
environmentVariableName: string,
environmentValue: string,
): void {
logger.trace(
`Adding ${environmentVariableName} environment variable for pipenv`,
);
if (
extraEnv[environmentVariableName] &&
extraEnv[environmentVariableName] !== environmentValue
) {
logger.warn(
`Possible misconfiguration, ${environmentVariableName} is already set to a different value`,
);
}
extraEnv[environmentVariableName] = environmentValue;
}

// Only one source is currently supported
return sourceWithCredentials ?? null;
/**
* Pipenv allows configuring source-urls for remote repositories with placeholders for credentials, i.e. http://$USER:$PASS@myprivate.repo
* if a matching host rule exists for that repository, we need to set the corresponding variables.
* Simply substituting them in the URL is not an option as it would impact the hash for the resulting Pipfile.lock
*
*/
async function addCredentialsForSourceUrls(
newPipfileContent: string,
pipfileName: string,
extraEnv: ExtraEnv<unknown>,
): Promise<void> {
const sourceUrls = await findPipfileSourceUrlsWithCredentials(
newPipfileContent,
pipfileName,
);
for (const parsedSourceUrl of sourceUrls) {
logger.trace(`Trying to add credentials for ${parsedSourceUrl.toString()}`);
const matchingHostRule = getMatchingHostRule(parsedSourceUrl.toString());
if (matchingHostRule) {
const usernameVariableName = extractEnvironmentVariableName(
parsedSourceUrl.username,
);
if (matchingHostRule.username && usernameVariableName) {
addExtraEnvVariable(
extraEnv,
usernameVariableName,
matchingHostRule.username,
);
}
const passwordVariableName = extractEnvironmentVariableName(
parsedSourceUrl.password,
);
if (matchingHostRule.password && passwordVariableName) {
addExtraEnvVariable(
extraEnv,
passwordVariableName,
matchingHostRule.password,
);
}
}
}
}

export async function updateArtifacts({
Expand Down Expand Up @@ -188,26 +258,7 @@ export async function updateArtifacts({
},
],
};

const sourceUrl = await findPipfileSourceUrlWithCredentials(
newPipfileContent,
pipfileName,
);
if (sourceUrl) {
logger.debug({ sourceUrl }, 'Pipfile contains credentials');
const hostRule = getMatchingHostRule(sourceUrl);
if (hostRule) {
logger.debug('Found matching hostRule for Pipfile credentials');
if (hostRule.username) {
logger.debug('Adding USERNAME environment variable for pipenv');
extraEnv.USERNAME = hostRule.username;
}
if (hostRule.password) {
logger.debug('Adding PASSWORD environment variable for pipenv');
extraEnv.PASSWORD = hostRule.password;
}
}
}
await addCredentialsForSourceUrls(newPipfileContent, pipfileName, extraEnv);
execOptions.extraEnv = extraEnv;

logger.trace({ cmd }, 'pipenv lock command');
Expand Down

0 comments on commit 6ae3818

Please sign in to comment.