Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Glue in backup copy of github deps for generated hex files #9529

Draft
wants to merge 10 commits into
base: master
Choose a base branch
from
Draft

Glue in backup copy of github deps for generated hex files #9529

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
1be0b72
pack in external deps (pub:/github:) to generated hex files
jwunderl May 23, 2023
404d63a
Committing checkpoint to revert; this approach is messy, outlined bet…
jwunderl May 23, 2023
75b7d17
Revert "Committing checkpoint to revert; this approach is messy, outl…
jwunderl May 23, 2023
e0d47a0
pass backup script text thru to github store, mark, pack into 1 subfile
jwunderl May 24, 2023
6fc7ada
recursive check to handle nested gh deps
jwunderl May 24, 2023
b906ef1
don't return from finally since linter doesn't like it
jwunderl May 24, 2023
098381c
Merge branch 'master' of https://github.com/microsoft/pxt into glueIn…
jwunderl Jun 13, 2023
48d5e13
handle circ deps
jwunderl Jun 13, 2023
48d44ca
unwrap a bit so it's just deps, so hex info would be stored separately
jwunderl Jun 13, 2023
5e71e5e
plug in portion mocking out where to read / write hex to / from
jwunderl Jun 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
120 changes: 77 additions & 43 deletions pxtlib/github.ts
View file
Open in desktop
Expand Up @@ -101,13 +101,14 @@ namespace pxt.github {

export interface CachedPackage {
files: Map<string>;
backupCopy?: boolean;
}

// caching
export interface IGithubDb {
latestVersionAsync(repopath: string, config: PackagesConfig): Promise<string>;
loadConfigAsync(repopath: string, tag: string): Promise<pxt.PackageConfig>;
loadPackageAsync(repopath: string, tag: string): Promise<CachedPackage>;
loadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map<string>): Promise<CachedPackage>;
}

function ghRequestAsync(options: U.HttpRequestOptions) {
Expand Down Expand Up @@ -168,19 +169,19 @@ namespace pxt.github {
private configs: pxt.Map<pxt.PackageConfig> = {};
private packages: pxt.Map<CachedPackage> = {};

private proxyWithCdnLoadPackageAsync(repopath: string, tag: string): Promise<CachedPackage> {
private async proxyWithCdnLoadPackageAsync(repopath: string, tag: string): Promise<CachedPackage> {
// cache lookup
const key = `${repopath}/${tag}`;
let res = this.packages[key];
if (res) {
pxt.debug(`github cache ${repopath}/${tag}/text`);
return Promise.resolve(res);
return res;
}

// load and cache
const parsed = parseRepoId(repopath)
return ghProxyWithCdnJsonAsync(join(parsed.slug, tag, parsed.fileName, "text"))
.then(v => this.packages[key] = { files: v });
const parsed = parseRepoId(repopath);
const v = await ghProxyWithCdnJsonAsync(join(parsed.slug, tag, parsed.fileName, "text"));
return this.packages[key] = { files: v };
}

private cacheConfig(key: string, v: string) {
Expand Down Expand Up @@ -227,7 +228,7 @@ namespace pxt.github {
return resolved
}

async loadPackageAsync(repopath: string, tag: string): Promise<CachedPackage> {
async loadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map<string>): Promise<CachedPackage> {
if (!tag) {
pxt.debug(`load pkg: default to master branch`)
tag = "master";
Expand All @@ -243,41 +244,48 @@ namespace pxt.github {
}

// try using github apis
return await this.githubLoadPackageAsync(repopath, tag);
return await this.githubLoadPackageAsync(repopath, tag, backupScriptText);
}

private githubLoadPackageAsync(repopath: string, tag: string): Promise<CachedPackage> {
return tagToShaAsync(repopath, tag)
.then(sha => {
// cache lookup
const key = `${repopath}/${sha}`;
let res = this.packages[key];
if (res) {
pxt.debug(`github cache ${repopath}/${tag}/text`);
return Promise.resolve(U.clone(res));
private async githubLoadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map<string>): Promise<CachedPackage> {
// load and cache
const current: CachedPackage = {
files: {}
}
const key = `${repopath}/${tag}`;
// ^^ double check this diff; is there a reason to store keyed off sha? just for master special case maybe?
try {
const sha = await tagToShaAsync(repopath, tag);
// cache lookup
let res = this.packages[key];
if (res) {
pxt.debug(`github cache ${repopath}/${tag}/text`);
return U.clone(res);
}

pxt.log(`Downloading ${repopath}/${tag} -> ${sha}`);
const pkg = await downloadTextAsync(repopath, sha, pxt.CONFIG_NAME);
current.files[pxt.CONFIG_NAME] = pkg;
const cfg: pxt.PackageConfig = JSON.parse(pkg);
await U.promiseMapAll(
pxt.allPkgFiles(cfg).slice(1),
async fn => {
const text = await downloadTextAsync(repopath, sha, fn);
current.files[fn] = text;
}
)
} catch (e) {
if (backupScriptText) {
current.files = U.clone(backupScriptText);
current.backupCopy = true;
} else {
throw e;
}
}

// load and cache
pxt.log(`Downloading ${repopath}/${tag} -> ${sha}`)
return downloadTextAsync(repopath, sha, pxt.CONFIG_NAME)
.then(pkg => {
const current: CachedPackage = {
files: {}
}
current.files[pxt.CONFIG_NAME] = pkg
const cfg: pxt.PackageConfig = JSON.parse(pkg)
return U.promiseMapAll(pxt.allPkgFiles(cfg).slice(1),
fn => downloadTextAsync(repopath, sha, fn)
.then(text => {
current.files[fn] = text
}))
.then(() => {
// cache!
this.packages[key] = current;
return U.clone(current);
})
})
})
// cache!
this.packages[key] = current;
return U.clone(current);
}
}

Expand Down Expand Up @@ -586,7 +594,7 @@ namespace pxt.github {
return await db.loadConfigAsync(repopath, tag)
}

export async function downloadPackageAsync(repoWithTag: string, config: pxt.PackagesConfig): Promise<CachedPackage> {
export async function downloadPackageAsync(repoWithTag: string, config: pxt.PackagesConfig, backupScriptText?: pxt.Map<string>): Promise<CachedPackage> {
const p = parseRepoId(repoWithTag)
if (!p) {
pxt.log('Unknown GitHub syntax');
Expand All @@ -599,12 +607,13 @@ namespace pxt.github {
return undefined;
}

// TODO check if this needs adjustments / special casing for backupScriptText; try catch around it?
// always try to upgrade unbound versions
if (!p.tag) {
p.tag = await db.latestVersionAsync(p.slug, config)
}
const cached = await db.loadPackageAsync(p.fullName, p.tag)
const dv = upgradedDisablesVariants(config, repoWithTag)
const cached = await db.loadPackageAsync(p.fullName, p.tag, backupScriptText);
const dv = upgradedDisablesVariants(config, repoWithTag);
if (dv) {
const cfg = Package.parseAndValidConfig(cached.files[pxt.CONFIG_NAME])
if (cfg) {
Expand All @@ -630,21 +639,46 @@ namespace pxt.github {
return { version, config };
}

export async function cacheProjectDependenciesAsync(cfg: pxt.PackageConfig): Promise<void> {
export async function cacheProjectDependenciesAsync(
cfg: pxt.PackageConfig,
backupExtensions?: pxt.Map<pxt.Map<string>>
): Promise<void> {
return cacheProjectDependenciesAsyncCore(
cfg,
{} /** resolved */,
backupExtensions
);
}

async function cacheProjectDependenciesAsyncCore(
cfg: pxt.PackageConfig,
checked: pxt.Map<boolean>,
backupExtensions?: pxt.Map<pxt.Map<string>>
): Promise<void> {
const ghExtensions = Object.keys(cfg.dependencies)
?.filter(dep => isGithubId(cfg.dependencies[dep]));

// need to check/cache pub: links + inject to cache?
// probably fine to put off from initial pass as it's a bit of an edge case?
// maybe extend pxt.github.cache... a bit?
if (ghExtensions.length) {
const pkgConfig = await pxt.packagesConfigAsync();
// Make sure external packages load before installing header.
await Promise.all(
ghExtensions.map(
async ext => {
const extSrc = cfg.dependencies[ext];
const ghPkg = await downloadPackageAsync(extSrc, pkgConfig);
if (checked[extSrc])
return;
checked[extSrc] = true;
const backup = backupExtensions?.[extSrc];
const ghPkg = await downloadPackageAsync(extSrc, pkgConfig, backup);
if (!ghPkg) {
throw new Error(lf("Cannot load extension {0} from {1}", ext, extSrc));
}

const pkgCfg = pxt.U.jsonTryParse(ghPkg.files[pxt.CONFIG_NAME]);
await cacheProjectDependenciesAsyncCore(pkgCfg, checked, backupExtensions);
}
)
);
Expand Down
3 changes: 3 additions & 0 deletions pxtlib/main.ts
View file
Open in desktop
Expand Up @@ -512,6 +512,9 @@ namespace pxt {
export const TUTORIAL_CUSTOM_TS = "tutorial.custom.ts";
export const BREAKPOINT_TABLET = 991; // TODO (shakao) revisit when tutorial stuff is more settled
export const PALETTES_FILE = "_palettes.json";
// for packing extensions into distributables, as backup when network unavailable
export const PACKAGED_EXTENSIONS = "_packaged-extensions.json";
export const PACKAGED_EXT_INFO = "_packaged-ext-info.json";

export function outputName(trg: pxtc.CompileTarget = null) {
if (!trg) trg = appTarget.compile
Expand Down
85 changes: 64 additions & 21 deletions pxtlib/package.ts
View file
Open in desktop
Expand Up @@ -1220,8 +1220,7 @@ namespace pxt {
variants = [null]
}


let ext: pxtc.ExtensionInfo = null
let ext: pxtc.ExtensionInfo = null;
for (let v of variants) {
if (ext)
pxt.debug(`building for ${v}`)
Expand All @@ -1247,7 +1246,16 @@ namespace pxt {
!opts.target.isNative

if (!noFileEmbed) {
const files = await this.filesToBePublishedAsync(true)
// Include packages when it won't influence flash size.
const files = await this.filesToBePublishedAsync(
true,
!appTarget.compile.useUF2
);
if (opts.target.isNative && opts.extinfo.hexinfo) {
// todo trim down to relevant portion of extinfo?
// hexfile + hash + whatever is needed for /cpp.ts
files[pxt.PACKAGED_EXT_INFO] = JSON.stringify(opts.extinfo);
}
const headerString = JSON.stringify({
name: this.config.name,
comment: this.config.description,
Expand All @@ -1257,7 +1265,9 @@ namespace pxt {
editor: this.getPreferredEditor(),
targetVersions: pxt.appTarget.versions
})
const programText = JSON.stringify(files)

const programText = JSON.stringify(files);

const buf = await lzmaCompressAsync(headerString + programText)
if (buf) {
opts.embedMeta = JSON.stringify({
Expand All @@ -1268,7 +1278,7 @@ namespace pxt {
eURL: pxt.appTarget.appTheme.embedUrl,
eVER: pxt.appTarget.versions ? pxt.appTarget.versions.target : "",
pxtTarget: appTarget.id,
})
});
opts.embedBlob = ts.pxtc.encodeBase64(U.uint8ArrayToString(buf))
}
}
Expand Down Expand Up @@ -1314,24 +1324,57 @@ namespace pxt {
return cfg;
}

filesToBePublishedAsync(allowPrivate = false) {
async filesToBePublishedAsync(allowPrivate = false, packExternalExtensions = false) {
const files: Map<string> = {};
return this.loadAsync()
.then(() => {
if (!allowPrivate && !this.config.public)
U.userError('Only packages with "public":true can be published')
const cfg = this.prepareConfigToBePublished();
files[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(cfg);
for (let f of this.getFiles()) {
// already stored
if (f == pxt.CONFIG_NAME) continue;
let str = this.readFile(f)
if (str == null)
U.userError("referenced file missing: " + f)
files[f] = str
await this.loadAsync();
if (!allowPrivate && !this.config.public)
U.userError('Only packages with "public":true can be published')
const cfg = this.prepareConfigToBePublished();
files[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(cfg);

for (let f of this.getFiles()) {
// already stored
if (f == pxt.CONFIG_NAME) continue;
let str = this.readFile(f)
if (str == null)
U.userError("referenced file missing: " + f)
files[f] = str
}

if (packExternalExtensions) {
const packedDeps: Map<Map<string>> = {};
const packDeps = (p: Package) => {
const depsToPack = p.resolvedDependencies()
.filter(dep => {
switch (dep.verProtocol()) {
case "github":
case "pub":
return true;
default:
return false;
}
});

for (const dep of depsToPack) {
if (packedDeps[dep._verspec])
continue;
const packed: Map<string> = {};
for (const toPack of dep.getFiles()) {
packed[toPack] = dep.readFile(toPack);
}
packed[pxt.CONFIG_NAME] = JSON.stringify(dep.config);

packedDeps[dep._verspec] = packed;
packDeps(dep);
}
return U.sortObjectFields(files)
})
}

packDeps(this);
if (Object.keys(packedDeps).length) {
files[pxt.PACKAGED_EXTENSIONS] = JSON.stringify(packedDeps);
}
}
return U.sortObjectFields(files);
}

saveToJsonAsync(): Promise<pxt.cpp.HexFile> {
Expand Down
43 changes: 23 additions & 20 deletions webapp/src/db.ts
View file
Open in desktop
Expand Up @@ -122,32 +122,35 @@ class GithubDb implements pxt.github.IGithubDb {
} // not found
);
}
loadPackageAsync(repopath: string, tag: string): Promise<pxt.github.CachedPackage> {
async loadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map<string>): Promise<pxt.github.CachedPackage> {
if (!tag) {
pxt.debug(`dep: default to master`)
tag = "master"
pxt.debug(`dep: default to master`)
tag = "master"
}
// don't cache master
if (tag == "master")
return this.mem.loadPackageAsync(repopath, tag);
return this.mem.loadPackageAsync(repopath, tag, backupScriptText);

const id = `pkg-${repopath}-${tag}`;
return this.table.getAsync(id).then(
entry => {
pxt.debug(`github offline cache hit ${id}`);
return entry.package as pxt.github.CachedPackage;
},
e => {
pxt.debug(`github offline cache miss ${id}`);
return this.mem.loadPackageAsync(repopath, tag)
.then(p => {
return this.table.forceSetAsync({
id,
package: p
}).then(() => p, e => p);
})
} // not found
);
try {
const entry = await this.table.getAsync(id);
pxt.debug(`github offline cache hit ${id}`);
// TODO: back up check here if .backupCopy to try fetch from this.mem?
return entry.package as pxt.github.CachedPackage;

} catch (e) {
pxt.debug(`github offline cache miss ${id}`);
const p = await this.mem.loadPackageAsync(repopath, tag, backupScriptText);
try {
await this.table.forceSetAsync({
id,
package: p
});
} catch (e) {
// swallow caching error
}
return p;
}
}
}

Expand Down