diff --git a/README.md b/README.md index e61f927..0e00b2c 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,9 @@ container-image-builder without any docker dependency this is a container registry client and image builder with support for protocol version 2 -This library comes in 2 parts. +This library comes in a few parts. -- an Image class. which takes care of authentication and creating new registry clients. +- an Image class. which takes care of authentication, creating new registry clients, and packaging up local files. - registry clients. perform exactly the api calls defined in the [oci spec.](https://github.com/opencontainers/distribution-spec/blob/master/spec.md) you can do most things with the Image class. It takes care of tedious tasks like copying layers missing from one registry into another so you can save your new image. @@ -21,7 +21,7 @@ import {Image} from 'container-image-builder' const image = new Image('node:lts-slim','gcr.io/my-project/my-image') // add local files to the image. -await image.addFiles("./",'/directory-root-for-files-in-container') +await image.addFiles({"./":'/directory-root-for-files-in-container'}) // creates a layer in the image with files like // /directory-root-for-files-in-container/[the files in ./] @@ -30,7 +30,7 @@ await image.addFiles("./",'/directory-root-for-files-in-container') image.WorkingDir = "/directory-root-for-files-in-container" // set the default command to run in the container image.Cmd = ["node","index.js"] -// set environment variables. does not remove old env vars. +// append environment variables. image.Env = [] // save the new image at latest @@ -53,31 +53,243 @@ docker run gcr.io/my-project/my-image:latest npm install container-image-builder ``` +### terms + +Learning about this distribution api I struggled mapping the names of some of the things to things i was familiar with. + +defined in the order that they compose into an "Image" + +- digest + - sha256 sum encoded as hex prefixed with "sha256:" + +- blob + - any file stored in a docker registry. + - but they are mostly 2 kinds. "config" json or "layer" tarball + +- layer + - is a tarball. + - these tarballs are expanded at the root of the file system when run as part of an image as a container. + +- config + - a json document stored as a blob and referenced in the manifest + - this hold details about the environment and what commands should be run when your image run as a container. + - this holds an array of the shasums of every layer in a property called diff_ids. I call this uncompressedDigest throughout this client. + - i call it uncompressedDigest because its the shasum of the layer before it's gzipped. + - because we need the uncompressed digest for every layer quite a few things in this api are less straight forward that you might expect. + +- manifest + - a json document that stores information about all of the layers in the image, and the digest of the config. + - each layer is a reference to a blob by digest along with contentLength and other optional properties. + - a manifest can have a tag which is a convenient way to name a specific version of the image. + +- Image + - is a combination of N layer blobs + - one image config blob + - and the manifest + +- Container + - a process running an Image. + - docker and other container runners make images useful. + ## API -- `require('container-image-builder').Image` or `import {Image} from 'container-image-builder'` +- `const {Image} = require('container-image-builder')` +- or `import {Image} from 'container-image-builder'` in typescript etc. ### Image builder API -- image = new Image(baseImage:string,targetImage:string) +- `image = new Image(baseImage:string,targetImage:string)` - baseImage the name of the base image. these are image specifiers just like you would pass to other container tools. - targetImage the name of the image you're going to be saving to. calls to image.save() will replace this image. -- image.addFiles(localDirectory:string,targetDirectory:string) :Promise<..> - - tar a local directory as a layer and place it at targetDirectory in the image - - does not support symlinks outside of the localDirectory. +- `image.addFiles({[localDirectory]:targetDirectory},options) :Promise<..>` + - tar local directories and place each at targetDirectory in a single layer + - symlinks are replaced with their files/directories as they are written to the layer tarball by default. + - options + - `options.ignores` + - optional set of globs applied at the root of the local directory processed by [micromatch](https://www.npmjs.com/package/micromatch) + - `options.ignoreFiles` + - optional array of file names to be parsed for ignore globs. like `[".ignore"]` + - `options.tar` + - control how the tar is packed with options from [node tar](https://www.npmjs.com/package/tar#class-tarpack) + - control directory traversal with options from [walkdir](https://www.npmjs.com/package/walkdir) + + - you can also call it with arguments like this but you should probably use the default. + - `image.addFiles(localDirectory,targetDirectory,options) :Promise<..>` + - `image.addFiles(localDirectory,options) :Promise<..>` + +- `image.save(tags?: string[], options)` + - save changes to the image. by default this saves the updated image as the `latest` tag + - `tags`, string[] + - if you specify tags the manifest will be taged with each of them instead of the default. + - `options` + - Cmd. see image.Cmd below + - Env. see image.Env below + - WorkingDir. see image.WorkingDir below + - copyRemoteLayers, default true + - set to false if you use nondistributable layers and cannot copy them to the target registry. + - layers with a urls field will be copied into the target registry by default from the first url to return a 200 in the array. + - ignoreExists, default false. + - primarily for debug/testing. this copies a blob into the target registry even though it already exists in the target + +- `image.WorkingDir = "/custom working dir"` + - string. + - If set this set the container's default working directory for commands. + +- `image.Cmd = ["ls","./"]` + - string[] + - If set this will replace the base images default command with the one you specify. + +- `image.Env = ["HOME=/workspace"]` + - string[] + - append environment variables to the base image's env vars + +- `image.getImageConfig() Promise<{ImageConfig}>` + - returns reference to the part of the image config you're most likely to want to change. + - changes to this object are saved when you call `image.save()` + +- `image.getImageData() Promise<{manifest:ImageManifest,config:ImageConfig}>` + - returns internal copies of the image manifest and config + - you need this if you are doing fancy things like replacing a specific layer + - _warning_ its easy to make changes here that will create an image that does not work. + - changes to these objects will be saved when you call `image.save()` + +- `image.client(imageSpecifier,writePermission) Promise` + - args are optional. uses a cached client if one already exists. + - return a promise to an authenticated registry client. + - auth is performed with options passed to new Image. + - imageSpecifier, string + - just like you would pass to docker or the image constructor. + - default is the base image used in the constructor + +- `image.sync(options)` + - options. optional + - copyRemoteLayers, default true + - set to false if you use nondistributable layers and cannot copy them to the target registry. + - layers with a urls field will be copied into the target registry by default from the first url to return a 200 in the array. + - ignoreExists, default false. + - primarily for debug/testing. this copies a blob into the target registry even though it already exists in the target + - manually trigger a sync from base image to target image registry. + - this copies blobs from one registry to another. you may want to use this for performance to start copying base image layers while you do other work. + - if the image has been synced it will not be synced in the call to `image.save()` + +- `image.addLayer(digest: string, uncompressedDigest: string, size: number,urls?: string[])` + - digest _required_ + - this is the id for the layer in the manifest + - uncompressedDigest _required_ + - this is the id for the layer in the diff_ids array from the image config that maps to the layer + - size _required_ + - the number of bytes of the compressed layer tarball. + - urls, optional + - string[] of urls where we can download this layer tarball. used mostly for nondistributable layers. + +- `image.removeLayer(digest:string)` + - digest, string _required_ + - the sha256 sum of the compressed layer tarball from a manifest. + + - remove the layer tagged in the manifest by digest. save it's offset in the array. + - remove the uncompressedDigest from the image config that matches the offset above + +### docker registry auth + +`const {auth} = require('container-image-builder')` + +authenticate to docker registries. This library has built in support for gcr.io and read only access to docker hub. +for all other cases it'll fall back to using docker credential helpers already installed on your system. + +- `auth(imageSpecifier,scope,options) Promise<{Secret?:string,Username?:string,token?:string}>` + - imageSpecifier, string + - image location like you pass to docker + - scope, string "push" or "push,pull" + - passed to the auth api if requesting docker hub or gcr.io + - options + - options['gcr.io'] = {...} + - these are auth options passed directly to [google-auth-library](https://www.npmjs.com/package/google-auth-library) + - you can also set the environment variable `GOOGLE_APPLICATION_CREDENTIALS=path to key file.json` and it will work as expected like other google client libraries. + - options['docker.io'] = {...} + - accepts these options. all are strings. + - either `token` is required or `Username`,`Secret` is required. we'll either try Basic or Bearer auth depending on credentials provided. + - `options.Secret` + - `options.Username` + - `options.token` ### registry client API as you get more creative you'll find you have to combine work done in the image builder with lower level api calls. like adding a new blob directly to the target registry before you call addLayer. -todo + +`const RegistryClient = require('container-image-builder')` or +`import {RegistryClient} from 'container-image-builder` + +- `client = new RegistryClient(registry, repository, auth)` + - registry, string _required_ + - repository, string _required_ + - this is the image name including any namespace portion. so something like `my-google-project/my-app` + - auth _required_ + - auth.Username + - auth.Secret + or + - auth.token + +- `client.manifest(tag: string) Promise` + - note: get the image manifest + - tag, string _required_ + - this can be a tag name or a digest for manifests without tags. + +- `client.tags() Promise` + - note: list all tags associated with the image (repository) + - no arguments. + - TagResult + - child: any[]; + - manifest: {}; + - name: string; + - tags: string[]; + +- `client.manifestUpload(tag: string|false, manifest: Buffer|{}) Promise<{status: number, digest: string, body: Buffer}>` + - tag, string| false + - if ! tag manifest will be uploaded untagged. + - manifest + - if manifest is a buffer it'll be uploaded as is. + - if manifest is an object it'll be json.stringified and uploaded. + - this does no verification at all. the registry api will fail if anything is wrong with it. + +- `client.blobExists(digest) Promise` + - digest, string _required_ + - the sha256sum of the blob. + - both layers and image configs are stored as blobs in the registry api + +- `client.blob(digest, stream) Promise` + - note: download a blob as a buffer or stream + - digest, string _required_ + - the sha256 sum of the blob you want to download + - stream, boolean + - default false + - if you'ed like to download to a buffer or resolve to a readable stream. + +- `client.upload(blob, contentLength, digest) Promise<{contentLength: number, digest: string}> ` + - note: upload a blob to the registry. you do not need to know the content length and digest before hand. if they're not provided they'll be calculated on the fly and a 2 step upload will be performed. It's more efficient if you know the contentLength and digest beforehand, but if you're streaming it can be more efficient to calculate on the fly. + - blob, Readable|Buffer _required_ + - a readable stream or buffer + - contentLength, number optional + - the size in bytes of the blob + - digest, string optional + - sha256sum of the blob + +- `client.mount(digest: string, fromRepository: string)` + - note: likely incomplete. this is much faster than alternatives if it works =) + - mount api has a neat feature where if the mount fails it'll give you a redirect header with the download url but this doesnt have the fallback bit implemented. + - this also may need work around auth. in Image make a client to download from one and pipe to and upload on two. + - digest, string _required_ + - the sha256 sum of the target blob + - the repository to mount from. ## Common actions +TODO. recipes for common workflows here. + ### update env vars or exec path of a container ### add a layer to an image diff --git a/bin/copy.js b/bin/copy.js index 8444dc0..a5fab50 100644 --- a/bin/copy.js +++ b/bin/copy.js @@ -1,5 +1,5 @@ // @ts-check -let Image = require('./build/src').Image +let Image = require('../build/src').Image let from = process.argv[2] diff --git a/bin/show.js b/bin/show.js index 70bba73..5c531a8 100644 --- a/bin/show.js +++ b/bin/show.js @@ -1,5 +1,5 @@ // @ts-check -let Image = require('./build/src').Image +let Image = require('../build/src').Image let from = process.argv[2] diff --git a/package-lock.json b/package-lock.json index 0b772b3..24af637 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,5 +1,5 @@ { - "name": "container-registry-client", + "name": "container-image-builder", "version": "1.0.0", "lockfileVersion": 1, "requires": true, @@ -14,14 +14,14 @@ } }, "@babel/generator": { - "version": "7.3.0", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.3.0.tgz", - "integrity": "sha512-dZTwMvTgWfhmibq4V9X+LMf6Bgl7zAodRn9PvcPdhlzFMbvUutx74dbEv7Atz3ToeEpevYEJtAwfxq/bDCzHWg==", + "version": "7.3.3", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.3.3.tgz", + "integrity": "sha512-aEADYwRRZjJyMnKN7llGIlircxTCofm3dtV5pmY6ob18MSIuipHpA2yZWkPlycwu5HJcx/pADS3zssd8eY7/6A==", "dev": true, "requires": { - "@babel/types": "^7.3.0", + "@babel/types": "^7.3.3", "jsesc": "^2.5.1", - "lodash": "^4.17.10", + "lodash": "^4.17.11", "source-map": "^0.5.0", "trim-right": "^1.0.1" } @@ -75,9 +75,9 @@ } }, "@babel/parser": { - "version": "7.3.1", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.3.1.tgz", - "integrity": "sha512-ATz6yX/L8LEnC3dtLQnIx4ydcPxhLcoy9Vl6re00zb2w5lG6itY6Vhnr1KFRPq/FHNsgl/gh2mjNN20f9iJTTA==", + "version": "7.3.3", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.3.3.tgz", + "integrity": "sha512-xsH1CJoln2r74hR+y7cg2B5JCPaTh+Hd+EbBRk9nWGSNspuo6krjhX0Om6RnRQuIvFq8wVXCLKH3kwKDYhanSg==", "dev": true }, "@babel/template": { @@ -120,13 +120,13 @@ } }, "@babel/types": { - "version": "7.3.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.3.0.tgz", - "integrity": "sha512-QkFPw68QqWU1/RVPyBe8SO7lXbPfjtqAxRYQKpFpaB8yMq7X2qAqfwK5LKoQufEkSmO5NQ70O6Kc3Afk03RwXw==", + "version": "7.3.3", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.3.3.tgz", + "integrity": "sha512-2tACZ80Wg09UnPg5uGAOUvvInaqLk3l/IAhQzlxLQOIXacr6bMsra5SH6AWw/hIDRCSbCdHP2KzSOD+cT7TzMQ==", "dev": true, "requires": { "esutils": "^2.0.2", - "lodash": "^4.17.10", + "lodash": "^4.17.11", "to-fast-properties": "^2.0.0" } }, @@ -1286,9 +1286,9 @@ } }, "globals": { - "version": "11.10.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-11.10.0.tgz", - "integrity": "sha512-0GZF1RiPKU97IHUO5TORo9w1PwrH/NBPl+fS7oMLdaTRiYmYbwK4NWoZWrAdd0/abG9R2BU+OiwyQpTpE6pdfQ==", + "version": "11.11.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.11.0.tgz", + "integrity": "sha512-WHq43gS+6ufNOEqlrDBxVEbb8ntfXrfAUU2ZOpCxrBdGKW3gyv8mCxAfIBD0DroPKGrJ2eSsXsLtY9MPntsyTw==", "dev": true }, "google-auth-library": { @@ -1757,15 +1757,15 @@ "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, "istanbul-lib-coverage": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.1.tgz", - "integrity": "sha512-nPvSZsVlbG9aLhZYaC3Oi1gT/tpyo3Yt5fNyf6NmcKIayz4VV/txxJFFKAK/gU4dcNn8ehsanBbVHVl0+amOLA==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.3.tgz", + "integrity": "sha512-dKWuzRGCs4G+67VfW9pBFFz2Jpi4vSp/k7zBcJ888ofV5Mi1g5CUML5GvMvV6u9Cjybftu+E8Cgp+k0dI1E5lw==", "dev": true }, "istanbul-lib-instrument": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-3.0.0.tgz", - "integrity": "sha512-eQY9vN9elYjdgN9Iv6NS/00bptm02EBBk70lRMaVjeA6QYocQgenVrSgC28TJurdnZa80AGO3ASdFN+w/njGiQ==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-3.1.0.tgz", + "integrity": "sha512-ooVllVGT38HIk8MxDj/OIHXSYvH+1tq/Vb38s8ixt9GoJadXska4WkGY+0wkmtYCZNYtaARniH/DixUGGLZ0uA==", "dev": true, "requires": { "@babel/generator": "^7.0.0", @@ -1773,7 +1773,7 @@ "@babel/template": "^7.0.0", "@babel/traverse": "^7.0.0", "@babel/types": "^7.0.0", - "istanbul-lib-coverage": "^2.0.1", + "istanbul-lib-coverage": "^2.0.3", "semver": "^5.5.0" } }, @@ -2236,54 +2236,37 @@ } }, "nyc": { - "version": "13.1.0", - "resolved": "https://registry.npmjs.org/nyc/-/nyc-13.1.0.tgz", - "integrity": "sha512-3GyY6TpQ58z9Frpv4GMExE1SV2tAgYqC7HSy2omEhNiCT3mhT9NyiOvIE8zkbuJVFzmvvNTnE4h/7/wQae7xLg==", + "version": "13.3.0", + "resolved": "https://registry.npmjs.org/nyc/-/nyc-13.3.0.tgz", + "integrity": "sha512-P+FwIuro2aFG6B0Esd9ZDWUd51uZrAEoGutqZxzrVmYl3qSfkLgcQpBPBjtDFsUQLFY1dvTQJPOyeqr8S9GF8w==", "dev": true, "requires": { "archy": "^1.0.0", "arrify": "^1.0.1", - "caching-transform": "^2.0.0", + "caching-transform": "^3.0.1", "convert-source-map": "^1.6.0", - "debug-log": "^1.0.1", "find-cache-dir": "^2.0.0", "find-up": "^3.0.0", "foreground-child": "^1.5.6", "glob": "^7.1.3", - "istanbul-lib-coverage": "^2.0.1", - "istanbul-lib-hook": "^2.0.1", - "istanbul-lib-instrument": "^3.0.0", - "istanbul-lib-report": "^2.0.2", - "istanbul-lib-source-maps": "^2.0.1", - "istanbul-reports": "^2.0.1", + "istanbul-lib-coverage": "^2.0.3", + "istanbul-lib-hook": "^2.0.3", + "istanbul-lib-instrument": "^3.1.0", + "istanbul-lib-report": "^2.0.4", + "istanbul-lib-source-maps": "^3.0.2", + "istanbul-reports": "^2.1.1", "make-dir": "^1.3.0", "merge-source-map": "^1.1.0", "resolve-from": "^4.0.0", - "rimraf": "^2.6.2", + "rimraf": "^2.6.3", "signal-exit": "^3.0.2", "spawn-wrap": "^1.4.2", - "test-exclude": "^5.0.0", + "test-exclude": "^5.1.0", "uuid": "^3.3.2", - "yargs": "11.1.0", - "yargs-parser": "^9.0.2" + "yargs": "^12.0.5", + "yargs-parser": "^11.1.1" }, "dependencies": { - "align-text": { - "version": "0.1.4", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "kind-of": "^3.0.2", - "longest": "^1.0.1", - "repeat-string": "^1.5.2" - } - }, - "amdefine": { - "version": "1.0.1", - "bundled": true, - "dev": true - }, "ansi-regex": { "version": "3.0.0", "bundled": true, @@ -2308,9 +2291,12 @@ "dev": true }, "async": { - "version": "1.5.2", + "version": "2.6.2", "bundled": true, - "dev": true + "dev": true, + "requires": { + "lodash": "^4.17.11" + } }, "balanced-match": { "version": "1.0.0", @@ -2326,55 +2312,30 @@ "concat-map": "0.0.1" } }, - "builtin-modules": { - "version": "1.1.1", - "bundled": true, - "dev": true - }, "caching-transform": { - "version": "2.0.0", + "version": "3.0.1", "bundled": true, "dev": true, "requires": { - "make-dir": "^1.0.0", - "md5-hex": "^2.0.0", - "package-hash": "^2.0.0", - "write-file-atomic": "^2.0.0" + "hasha": "^3.0.0", + "make-dir": "^1.3.0", + "package-hash": "^3.0.0", + "write-file-atomic": "^2.3.0" } }, "camelcase": { - "version": "1.2.1", - "bundled": true, - "dev": true, - "optional": true - }, - "center-align": { - "version": "0.1.3", + "version": "5.0.0", "bundled": true, - "dev": true, - "optional": true, - "requires": { - "align-text": "^0.1.3", - "lazy-cache": "^1.0.3" - } + "dev": true }, "cliui": { - "version": "2.1.0", + "version": "4.1.0", "bundled": true, "dev": true, - "optional": true, "requires": { - "center-align": "^0.1.1", - "right-align": "^0.1.1", - "wordwrap": "0.0.2" - }, - "dependencies": { - "wordwrap": { - "version": "0.0.2", - "bundled": true, - "dev": true, - "optional": true - } + "string-width": "^2.1.1", + "strip-ansi": "^4.0.0", + "wrap-ansi": "^2.0.0" } }, "code-point-at": { @@ -2382,6 +2343,12 @@ "bundled": true, "dev": true }, + "commander": { + "version": "2.17.1", + "bundled": true, + "dev": true, + "optional": true + }, "commondir": { "version": "1.0.1", "bundled": true, @@ -2410,18 +2377,13 @@ } }, "debug": { - "version": "3.1.0", + "version": "4.1.1", "bundled": true, "dev": true, "requires": { - "ms": "2.0.0" + "ms": "^2.1.1" } }, - "debug-log": { - "version": "1.0.1", - "bundled": true, - "dev": true - }, "decamelize": { "version": "1.2.0", "bundled": true, @@ -2435,6 +2397,14 @@ "strip-bom": "^3.0.0" } }, + "end-of-stream": { + "version": "1.4.1", + "bundled": true, + "dev": true, + "requires": { + "once": "^1.4.0" + } + }, "error-ex": { "version": "1.3.2", "bundled": true, @@ -2449,12 +2419,12 @@ "dev": true }, "execa": { - "version": "0.7.0", + "version": "1.0.0", "bundled": true, "dev": true, "requires": { - "cross-spawn": "^5.0.1", - "get-stream": "^3.0.0", + "cross-spawn": "^6.0.0", + "get-stream": "^4.0.0", "is-stream": "^1.1.0", "npm-run-path": "^2.0.0", "p-finally": "^1.0.0", @@ -2463,11 +2433,13 @@ }, "dependencies": { "cross-spawn": { - "version": "5.1.0", + "version": "6.0.5", "bundled": true, "dev": true, "requires": { - "lru-cache": "^4.0.1", + "nice-try": "^1.0.4", + "path-key": "^2.0.1", + "semver": "^5.5.0", "shebang-command": "^1.2.0", "which": "^1.2.9" } @@ -2512,9 +2484,12 @@ "dev": true }, "get-stream": { - "version": "3.0.0", + "version": "4.1.0", "bundled": true, - "dev": true + "dev": true, + "requires": { + "pump": "^3.0.0" + } }, "glob": { "version": "7.1.3", @@ -2530,28 +2505,25 @@ } }, "graceful-fs": { - "version": "4.1.11", + "version": "4.1.15", "bundled": true, "dev": true }, "handlebars": { - "version": "4.0.11", + "version": "4.1.0", "bundled": true, "dev": true, "requires": { - "async": "^1.4.0", + "async": "^2.5.0", "optimist": "^0.6.1", - "source-map": "^0.4.4", - "uglify-js": "^2.6" + "source-map": "^0.6.1", + "uglify-js": "^3.1.4" }, "dependencies": { "source-map": { - "version": "0.4.4", + "version": "0.6.1", "bundled": true, - "dev": true, - "requires": { - "amdefine": ">=0.0.4" - } + "dev": true } } }, @@ -2560,6 +2532,14 @@ "bundled": true, "dev": true }, + "hasha": { + "version": "3.0.0", + "bundled": true, + "dev": true, + "requires": { + "is-stream": "^1.0.1" + } + }, "hosted-git-info": { "version": "2.7.1", "bundled": true, @@ -2585,7 +2565,7 @@ "dev": true }, "invert-kv": { - "version": "1.0.0", + "version": "2.0.0", "bundled": true, "dev": true }, @@ -2594,20 +2574,6 @@ "bundled": true, "dev": true }, - "is-buffer": { - "version": "1.1.6", - "bundled": true, - "dev": true, - "optional": true - }, - "is-builtin-module": { - "version": "1.0.0", - "bundled": true, - "dev": true, - "requires": { - "builtin-modules": "^1.0.0" - } - }, "is-fullwidth-code-point": { "version": "2.0.0", "bundled": true, @@ -2624,12 +2590,12 @@ "dev": true }, "istanbul-lib-coverage": { - "version": "2.0.1", + "version": "2.0.3", "bundled": true, "dev": true }, "istanbul-lib-hook": { - "version": "2.0.1", + "version": "2.0.3", "bundled": true, "dev": true, "requires": { @@ -2637,22 +2603,32 @@ } }, "istanbul-lib-report": { - "version": "2.0.2", + "version": "2.0.4", "bundled": true, "dev": true, "requires": { - "istanbul-lib-coverage": "^2.0.1", + "istanbul-lib-coverage": "^2.0.3", "make-dir": "^1.3.0", - "supports-color": "^5.4.0" + "supports-color": "^6.0.0" + }, + "dependencies": { + "supports-color": { + "version": "6.1.0", + "bundled": true, + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } + } } }, "istanbul-lib-source-maps": { - "version": "2.0.1", + "version": "3.0.2", "bundled": true, "dev": true, "requires": { - "debug": "^3.1.0", - "istanbul-lib-coverage": "^2.0.1", + "debug": "^4.1.1", + "istanbul-lib-coverage": "^2.0.3", "make-dir": "^1.3.0", "rimraf": "^2.6.2", "source-map": "^0.6.1" @@ -2666,11 +2642,11 @@ } }, "istanbul-reports": { - "version": "2.0.1", + "version": "2.1.1", "bundled": true, "dev": true, "requires": { - "handlebars": "^4.0.11" + "handlebars": "^4.1.0" } }, "json-parse-better-errors": { @@ -2678,27 +2654,12 @@ "bundled": true, "dev": true }, - "kind-of": { - "version": "3.2.2", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "is-buffer": "^1.1.5" - } - }, - "lazy-cache": { - "version": "1.0.4", - "bundled": true, - "dev": true, - "optional": true - }, "lcid": { - "version": "1.0.0", + "version": "2.0.0", "bundled": true, "dev": true, "requires": { - "invert-kv": "^1.0.0" + "invert-kv": "^2.0.0" } }, "load-json-file": { @@ -2721,19 +2682,18 @@ "path-exists": "^3.0.0" } }, - "lodash.flattendeep": { - "version": "4.4.0", + "lodash": { + "version": "4.17.11", "bundled": true, "dev": true }, - "longest": { - "version": "1.0.1", + "lodash.flattendeep": { + "version": "4.4.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "lru-cache": { - "version": "4.1.3", + "version": "4.1.5", "bundled": true, "dev": true, "requires": { @@ -2749,25 +2709,22 @@ "pify": "^3.0.0" } }, - "md5-hex": { - "version": "2.0.0", + "map-age-cleaner": { + "version": "0.1.3", "bundled": true, "dev": true, "requires": { - "md5-o-matic": "^0.1.1" + "p-defer": "^1.0.0" } }, - "md5-o-matic": { - "version": "0.1.1", - "bundled": true, - "dev": true - }, "mem": { - "version": "1.1.0", + "version": "4.1.0", "bundled": true, "dev": true, "requires": { - "mimic-fn": "^1.0.0" + "map-age-cleaner": "^0.1.1", + "mimic-fn": "^1.0.0", + "p-is-promise": "^2.0.0" } }, "merge-source-map": { @@ -2819,17 +2776,22 @@ } }, "ms": { - "version": "2.0.0", + "version": "2.1.1", + "bundled": true, + "dev": true + }, + "nice-try": { + "version": "1.0.5", "bundled": true, "dev": true }, "normalize-package-data": { - "version": "2.4.0", + "version": "2.5.0", "bundled": true, "dev": true, "requires": { "hosted-git-info": "^2.1.4", - "is-builtin-module": "^1.0.0", + "resolve": "^1.10.0", "semver": "2 || 3 || 4 || 5", "validate-npm-package-license": "^3.0.1" } @@ -2870,23 +2832,33 @@ "dev": true }, "os-locale": { - "version": "2.1.0", + "version": "3.1.0", "bundled": true, "dev": true, "requires": { - "execa": "^0.7.0", - "lcid": "^1.0.0", - "mem": "^1.1.0" + "execa": "^1.0.0", + "lcid": "^2.0.0", + "mem": "^4.0.0" } }, + "p-defer": { + "version": "1.0.0", + "bundled": true, + "dev": true + }, "p-finally": { "version": "1.0.0", "bundled": true, "dev": true }, - "p-limit": { + "p-is-promise": { "version": "2.0.0", "bundled": true, + "dev": true + }, + "p-limit": { + "version": "2.1.0", + "bundled": true, "dev": true, "requires": { "p-try": "^2.0.0" @@ -2906,13 +2878,13 @@ "dev": true }, "package-hash": { - "version": "2.0.0", + "version": "3.0.0", "bundled": true, "dev": true, "requires": { - "graceful-fs": "^4.1.11", + "graceful-fs": "^4.1.15", + "hasha": "^3.0.0", "lodash.flattendeep": "^4.4.0", - "md5-hex": "^2.0.0", "release-zalgo": "^1.0.0" } }, @@ -2940,6 +2912,11 @@ "bundled": true, "dev": true }, + "path-parse": { + "version": "1.0.6", + "bundled": true, + "dev": true + }, "path-type": { "version": "3.0.0", "bundled": true, @@ -2966,6 +2943,15 @@ "bundled": true, "dev": true }, + "pump": { + "version": "3.0.0", + "bundled": true, + "dev": true, + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "read-pkg": { "version": "3.0.0", "bundled": true, @@ -2993,12 +2979,6 @@ "es6-error": "^4.0.1" } }, - "repeat-string": { - "version": "1.6.1", - "bundled": true, - "dev": true, - "optional": true - }, "require-directory": { "version": "2.1.1", "bundled": true, @@ -3009,26 +2989,25 @@ "bundled": true, "dev": true }, - "resolve-from": { - "version": "4.0.0", - "bundled": true, - "dev": true - }, - "right-align": { - "version": "0.1.3", + "resolve": { + "version": "1.10.0", "bundled": true, "dev": true, - "optional": true, "requires": { - "align-text": "^0.1.1" + "path-parse": "^1.0.6" } }, + "resolve-from": { + "version": "4.0.0", + "bundled": true, + "dev": true + }, "rimraf": { - "version": "2.6.2", + "version": "2.6.3", "bundled": true, "dev": true, "requires": { - "glob": "^7.0.5" + "glob": "^7.1.3" } }, "safe-buffer": { @@ -3037,7 +3016,7 @@ "dev": true }, "semver": { - "version": "5.5.0", + "version": "5.6.0", "bundled": true, "dev": true }, @@ -3064,12 +3043,6 @@ "bundled": true, "dev": true }, - "source-map": { - "version": "0.5.7", - "bundled": true, - "dev": true, - "optional": true - }, "spawn-wrap": { "version": "1.4.2", "bundled": true, @@ -3084,7 +3057,7 @@ } }, "spdx-correct": { - "version": "3.0.0", + "version": "3.1.0", "bundled": true, "dev": true, "requires": { @@ -3093,7 +3066,7 @@ } }, "spdx-exceptions": { - "version": "2.1.0", + "version": "2.2.0", "bundled": true, "dev": true }, @@ -3107,7 +3080,7 @@ } }, "spdx-license-ids": { - "version": "3.0.0", + "version": "3.0.3", "bundled": true, "dev": true }, @@ -3138,16 +3111,8 @@ "bundled": true, "dev": true }, - "supports-color": { - "version": "5.4.0", - "bundled": true, - "dev": true, - "requires": { - "has-flag": "^3.0.0" - } - }, "test-exclude": { - "version": "5.0.0", + "version": "5.1.0", "bundled": true, "dev": true, "requires": { @@ -3158,43 +3123,30 @@ } }, "uglify-js": { - "version": "2.8.29", + "version": "3.4.9", "bundled": true, "dev": true, "optional": true, "requires": { - "source-map": "~0.5.1", - "uglify-to-browserify": "~1.0.0", - "yargs": "~3.10.0" + "commander": "~2.17.1", + "source-map": "~0.6.1" }, "dependencies": { - "yargs": { - "version": "3.10.0", + "source-map": { + "version": "0.6.1", "bundled": true, "dev": true, - "optional": true, - "requires": { - "camelcase": "^1.0.2", - "cliui": "^2.1.0", - "decamelize": "^1.0.0", - "window-size": "0.1.0" - } + "optional": true } } }, - "uglify-to-browserify": { - "version": "1.0.2", - "bundled": true, - "dev": true, - "optional": true - }, "uuid": { "version": "3.3.2", "bundled": true, "dev": true }, "validate-npm-package-license": { - "version": "3.0.3", + "version": "3.0.4", "bundled": true, "dev": true, "requires": { @@ -3215,12 +3167,6 @@ "bundled": true, "dev": true }, - "window-size": { - "version": "0.1.0", - "bundled": true, - "dev": true, - "optional": true - }, "wordwrap": { "version": "0.0.3", "bundled": true, @@ -3274,7 +3220,7 @@ "dev": true }, "write-file-atomic": { - "version": "2.3.0", + "version": "2.4.2", "bundled": true, "dev": true, "requires": { @@ -3284,7 +3230,7 @@ } }, "y18n": { - "version": "3.2.1", + "version": "4.0.0", "bundled": true, "dev": true }, @@ -3294,87 +3240,31 @@ "dev": true }, "yargs": { - "version": "11.1.0", + "version": "12.0.5", "bundled": true, "dev": true, "requires": { "cliui": "^4.0.0", - "decamelize": "^1.1.1", - "find-up": "^2.1.0", + "decamelize": "^1.2.0", + "find-up": "^3.0.0", "get-caller-file": "^1.0.1", - "os-locale": "^2.0.0", + "os-locale": "^3.0.0", "require-directory": "^2.1.1", "require-main-filename": "^1.0.1", "set-blocking": "^2.0.0", "string-width": "^2.0.0", "which-module": "^2.0.0", - "y18n": "^3.2.1", - "yargs-parser": "^9.0.2" - }, - "dependencies": { - "cliui": { - "version": "4.1.0", - "bundled": true, - "dev": true, - "requires": { - "string-width": "^2.1.1", - "strip-ansi": "^4.0.0", - "wrap-ansi": "^2.0.0" - } - }, - "find-up": { - "version": "2.1.0", - "bundled": true, - "dev": true, - "requires": { - "locate-path": "^2.0.0" - } - }, - "locate-path": { - "version": "2.0.0", - "bundled": true, - "dev": true, - "requires": { - "p-locate": "^2.0.0", - "path-exists": "^3.0.0" - } - }, - "p-limit": { - "version": "1.3.0", - "bundled": true, - "dev": true, - "requires": { - "p-try": "^1.0.0" - } - }, - "p-locate": { - "version": "2.0.0", - "bundled": true, - "dev": true, - "requires": { - "p-limit": "^1.1.0" - } - }, - "p-try": { - "version": "1.0.0", - "bundled": true, - "dev": true - } + "y18n": "^3.2.1 || ^4.0.0", + "yargs-parser": "^11.1.1" } }, "yargs-parser": { - "version": "9.0.2", + "version": "11.1.1", "bundled": true, "dev": true, "requires": { - "camelcase": "^4.1.0" - }, - "dependencies": { - "camelcase": { - "version": "4.1.0", - "bundled": true, - "dev": true - } + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" } } } diff --git a/package.json b/package.json index 06a9f53..92be03a 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "keywords": [], "scripts": { "test": "nyc mocha -t 30000 --require source-map-support/register build/test/*.js", + "test-one": "nyc mocha -t 30000 --require source-map-support/register $1", "check": "gts check", "clean": "gts clean", "compile": "tsc -p .", @@ -16,7 +17,7 @@ }, "author": "Ryan Day ", "license": "Apache 2.0", - "repository":"google/nodejs-container-image-builder", + "repository": "google/nodejs-container-image-builder", "devDependencies": { "@types/micromatch": "^3.1.0", "@types/mocha": "^5.2.5", @@ -25,7 +26,7 @@ "@types/request": "^2.48.1", "gts": "^0.9.0", "mocha": "^5.2.0", - "nyc": "^13.1.0", + "nyc": "^13.3.0", "source-map-support": "^0.5.10", "typescript": "~3.1.0" }, diff --git a/src/emitter.ts b/src/emitter.ts new file mode 100644 index 0000000..f87dfd0 --- /dev/null +++ b/src/emitter.ts @@ -0,0 +1,20 @@ +import * as events from 'events'; +import * as path from 'path'; +// global emitter for all your progress needs. +let globalEmitter: events.EventEmitter; + +export const emitter = () => { + if (!globalEmitter) { + globalEmitter = new events.EventEmitter(); + } + return globalEmitter; +}; + +export const logger = (file: string) => { + file = file.replace(path.dirname(__dirname), ''); + // tslint:disable-next-line:no-any + return (args: any[]) => { + if (!globalEmitter) return; + globalEmitter.emit('log', file, args); + }; +}; \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index 9344bbc..7bbcf29 100644 --- a/src/index.ts +++ b/src/index.ts @@ -15,19 +15,21 @@ import * as crypto from 'crypto'; import * as retry from 'p-retry'; import * as path from 'path'; -import {Readable, Stream} from 'stream'; -import {resolve} from 'url'; import * as zlib from 'zlib'; import {handler as dockerAuth} from './auth/dockerio'; import {handler as gcrAuth} from './auth/gcr'; import {DockerCredenitalHelpers} from './credentials-helper'; import {ImageLocation, parse as parseSpecifier} from './image-specifier'; +import * as packer from './packer'; import {pending, PendingTracker} from './pending'; -import {GcrClient, ImageConfig, ManifestV2} from './registry'; +import {ImageConfig, ManifestV2, RegistryClient} from './registry'; const tar = require('tar'); +// expose plain registry client. +export {RegistryClient} from './registry'; + export type ImageOptions = { auth?: AuthConfig, sync?: false|undefined @@ -37,6 +39,7 @@ export type ImageData = { config: ImageConfig }; + export class Image { private options: ImageOptions; private image: ImageLocation; @@ -45,7 +48,7 @@ export class Image { // the manifest and config for the source image private imageData: Promise; private originalManifest?: ManifestV2; - private clients: {[k: string]: GcrClient|Promise} = {}; + private clients: {[k: string]: RegistryClient|Promise} = {}; private pending: PendingTracker; @@ -123,18 +126,33 @@ export class Image { } // "./myfiles", "/workspace" // "localDirectory", "imageDirectory" - addFiles(dir: string, targetDir: string): Promise<{ + addFiles( + dir: string|{[dir: string]: string}, + targetDir?: string|packer.PackOptions, + options?: packer.PackOptions): Promise<{ mediaType: string, digest: string, size: number, uncompressedDigest: string }> { - dir = path.resolve(dir); + // dir,target,options + // dir,options + // {dir:target,....},options + if (typeof targetDir === 'string') { + if (typeof dir !== 'string') { + // addFiles({"apples":"oranges"},"pears") + throw new Error( + 'specifying a target directory name when the dir is an object of name:target doesn\'t make sense. try addFiles({dir:target})'); + } + dir = {[dir]: targetDir}; + } else if (targetDir) { + // options! + options = targetDir; + } + // have to wrap in promise because the tar stream can emit error out of band const p = new Promise(async (resolve, reject) => { - const tarStream = tar.c( - {cwd: path.resolve(dir), gzip: false, prefix: targetDir || ''}, - ['./']); + const tarStream = packer.pack(dir, options); tarStream.on('error', (e: Error) => reject(e)); @@ -188,7 +206,15 @@ export class Image { return {manifest, config}; } - client(image?: ImageLocation, write?: boolean) { + client(_image?: ImageLocation|string, write?: boolean) { + let image:ImageLocation; + if(typeof _image === 'string'){ + image = parseSpecifier(_image) + } else { + // typescript!!! + image = _image as ImageLocation + } + image = (image ? image : this.image); const scope = write ? 'push,pull' : 'pull'; let key = [image.registry, image.namespace, image.image].join(','); @@ -196,7 +222,6 @@ export class Image { const writeKey = key + ',push,pull'; const readKey = key + ',pull'; - // default to most permissive cached client even if it doesn't match scope. if (this.clients[writeKey]) { return this.clients[writeKey]; @@ -207,7 +232,7 @@ export class Image { key += ',' + scope; const promiseOfClient = auth(image, scope, this.options.auth || {}).then((registryAuth) => { - const registryClient = new GcrClient( + const registryClient = new RegistryClient( image!.registry, this.nameSpacedImageName(image), registryAuth); return registryClient; }); @@ -363,9 +388,14 @@ export class Image { export const auth = - async (image: ImageLocation, scope: string, options?: AuthConfig) => { + async (image: ImageLocation|string, scope: string, options?: AuthConfig) => { // todo: distinguish better between when we should try creds helpers vs only // built in. + + if(typeof image == 'string'){ + image = parseSpecifier(image) + } + try { if (image.registry.indexOf('gcr.io') > -1) { return await gcrAuth( diff --git a/src/packer.ts b/src/packer.ts index 5f84415..36a05b7 100644 --- a/src/packer.ts +++ b/src/packer.ts @@ -12,25 +12,178 @@ // See the License for the specific language governing permissions and // limitations under the License. -const tar = require('tar'); +import * as fs from 'fs'; +import * as _path from 'path'; +import {Readable} from 'stream'; + +import {logger} from './emitter'; import * as walker from './walker'; -import * as path from 'path'; -export const pack = (targetPath: string, dir: string, options: {}) => { - /* +const log = logger(__filename); + + +// had conversation with isaacs about making changes to node-tar so we dont have +// to do this. todo: link issue + +// tslint:disable-next-line:variable-name +const Header = require('tar/lib/header'); +// tslint:disable-next-line:variable-name +const ReadEntry = require('tar/lib/read-entry'); +// tslint:disable-next-line:variable-name +const Pack = require('tar/lib/pack.js'); +const modeFix = require('tar/lib/mode-fix'); + + +export type PackOptions = { + // tslint:disable-next-line:no-any + tar?: {[k: string]: any} +}&walker.Options; + +export const pack = + (paths: {[fromPath: string]: string}|string, options?: PackOptions) => { + // thanks typescript. + let pathObj: {[fromPath: string]: string} = {}; + if (typeof paths === 'string') { + pathObj[paths] = paths; + } else { + pathObj = paths; + } + + options = options || {}; + + // flatten every link into the tree its in + options.find_links = false; + options.no_return = true; + + let ends = 0; + let starts = 0; + + const queue: Array<{path: string, toPath: string, stat: fs.Stats}> = []; + + // tar gzip:false etc. + const pack = new Pack( + Object.assign({}, options.tar || {}, {gzip: false, jobs: Infinity})); + let working = false; + const work = () => { + if (working) return; + + const obj = queue.shift(); + if (!obj) { + if (walkEnded) { + pack.end(); + } + return; + } + working = true; + starts++; + let entry; + const {path, stat, toPath} = obj; + entry = pathToReadEntry({path, stat, toPath, portable: false}); + entry.on('end', () => { + ends++; + working = false; + work(); + }); + entry.on('error', (err: Error) => { + pack.emit('error', err); + }); + pack.write(entry); + }; + + + let walkEnded = false; + + const walks: Array> = []; + Object.keys(pathObj).forEach((path) => { + const toPath = pathObj[path]; + path = _path.resolve(path); + // ill need to use this to pause and resume. TODO + // tslint:disable-next-line:only-arrow-functions + walks.push(walker.walk(path, options, function(file, stat) { + queue.push({ + path: file, + toPath: _path.join(toPath, file.replace(path, '')), + stat, + }); + work(); + })); + }); + + Promise.all(walks) + .then(() => { + walkEnded = true; + if (!queue.length && !working) { + pack.end(); + } + }) + .catch((e) => { + pack.emit('error', e); + }); + + return pack as Readable; + }; + +function pathToReadEntry(opts: { + path: string, + toPath?: string, + linkpath?: string, stat: fs.Stats, + mtime?: number, + noMtime?: boolean, portable: boolean +}) { + const {path, linkpath, stat} = opts; + let {toPath} = opts; + if (!stat) { + throw new Error('stat missing for ' + opts); + } + + const myuid = process.getuid && process.getuid(); + const myuser = process.env.USER || ''; + + // settings. + // override mtime. + const mtime = opts.mtime; + // dont write an mtime + const noMtime = opts.noMtime; + // dont write anything other than size, linkpath, path and mode + const portable = opts.portable; + + // add trailing / to directory paths + toPath = toPath || path; + if (stat.isDirectory() && path.substr(-1) !== '/') { + toPath += '/'; + } + + const header = new Header({ + path: toPath, + // if this is a link. the path the link points to. + linkpath, + mode: modeFix(stat.mode, stat.isDirectory()), + uid: portable ? null : stat.uid, + gid: portable ? null : stat.gid, + size: stat.isDirectory() ? 0 : stat.size, + mtime: noMtime ? null : mtime || stat.mtime, + type: statToType(stat), + uname: portable ? null : stat.uid === myuid ? myuser : '', + atime: portable ? null : stat.atime, + ctime: portable ? null : stat.ctime + }); - let tars = dir.map((dir)=>{ - return tar.c({ - cwd:path.resolve(dir), - gzip:false, - prefix:targetPath||'' - }).pipe(tar.t(()=>{})); - }) + const entry = new ReadEntry(header); + if (stat.isFile()) { + fs.createReadStream(path).pipe(entry); + } else { + entry.end(); + } - let pack = new tar.Pack() + return entry; +} - tars.pipe - */ -}; +function statToType(stat: fs.Stats) { + if (stat.isDirectory()) return 'Directory'; + if (stat.isSymbolicLink()) return 'SymbolicLink'; + if (stat.isFile()) return 'File'; + // return nothing if unsupported. + return; +} diff --git a/src/registry.ts b/src/registry.ts index 7e88bd0..fc1a19b 100644 --- a/src/registry.ts +++ b/src/registry.ts @@ -13,22 +13,18 @@ // limitations under the License. // import * as crypto from 'crypto'; -import * as googleAuthLib from 'google-auth-library'; import {Response} from 'request'; import {Readable} from 'stream'; import * as urlModule from 'url'; -const gAuth = googleAuthLib.auth; - -// cant use teeny request because typings on return value. +// cant use teeny-request because typings on return value. import * as request from 'request'; import {DockerAuthResult} from './credentials-helper'; -// const request = require('hyperquest'); // https://docs.docker.com/registry/spec/api/ // https://github.com/opencontainers/distribution-spec/blob/master/spec.md -export class GcrClient { +export class RegistryClient { _auth: DockerAuthResult; _registry: string; _repository: string; diff --git a/src/walker.ts b/src/walker.ts index 416124b..01911f1 100644 --- a/src/walker.ts +++ b/src/walker.ts @@ -23,7 +23,9 @@ export type DirMap = { export type Options = walkdir.WalkOptions&{ignores?: string[], ignoreFiles?: string[]}; -export const walk = async (dir: string, options?: Options) => { +export const walk = async ( + dir: string, options?: Options, + onStat?: (path: string, stat: fs.Stats) => void) => { const cwd = process.cwd(); const entryDir = path.resolve(cwd, dir); options = options || {}; @@ -43,6 +45,8 @@ export const walk = async (dir: string, options?: Options) => { ignoreTree[entryDir] = ignores; } + console.log(ignoreTree); + const applyRules = (dir: string, files: string[]) => { const currentDir = dir; @@ -53,8 +57,6 @@ export const walk = async (dir: string, options?: Options) => { let i: number; while (dir.lastIndexOf(path.sep) > -1) { - i = dir.lastIndexOf(path.sep); - dir = dir.substr(0, i); if (ignoreTree[dir]) { files = files.filter((file) => { const relativeToRuleSource = @@ -67,6 +69,8 @@ export const walk = async (dir: string, options?: Options) => { return !micromatch.any(relativeToRuleSource, ignoreTree[dir]); }); } + i = dir.lastIndexOf(path.sep); + dir = dir.substr(0, i); } return files; @@ -76,7 +80,7 @@ export const walk = async (dir: string, options?: Options) => { if (!files.length) return []; const unread: Array> = []; - if (ignoreFiles.length) { + if (ignoreFiles.length || ignores.length) { // check each file name to see if it's an ignore file. files.forEach((name) => { if (ignoreFilesMap[name]) { @@ -108,7 +112,7 @@ export const walk = async (dir: string, options?: Options) => { }; } options.find_links = false; - return walkdir.async(entryDir, options); + return walkdir.async(entryDir, options, onStat); }; diff --git a/test/packer.ts b/test/packer.ts new file mode 100644 index 0000000..e731750 --- /dev/null +++ b/test/packer.ts @@ -0,0 +1,96 @@ +import * as assert from 'assert'; +import * as path from 'path'; +import {Readable} from 'stream'; + +import {pack} from '../src/packer'; + +describe('can pack', () => { + it('packs a directory', (done) => { + const nodeTar = require('tar'); + + const fixtureDir = path.join(__dirname, '..', '..', 'fixtures', 'project'); + + const tar = pack({[fixtureDir]: '/apples'}); + + let paths: string[] = []; + + const extract = tar.pipe(nodeTar.t()); + extract.on('entry', (e: Readable&{path: string}) => { + paths.push(e.path); + e.resume(); + }); + + extract.on('end', () => { + paths = paths.sort(); + assert.deepStrictEqual( + [ + 'apples/.ignore', 'apples/index.js', 'apples/lib/', + 'apples/lib/a-file.js', 'apples/test/', 'apples/test/taco.yaml', + 'apples/test/test.js' + ], + paths, 'should have tarred exactly the specified entries'); + + console.log(paths); + done(); + }); + }); + + it('packs a directory honoring an ignore file', (done) => { + const nodeTar = require('tar'); + + const fixtureDir = path.join(__dirname, '..', '..', 'fixtures', 'project'); + + const tar = pack({[fixtureDir]: '/apples'}, {ignoreFiles: ['.ignore']}); + + let paths: string[] = []; + + const extract = tar.pipe(nodeTar.t()); + extract.on('entry', (e: Readable&{path: string}) => { + paths.push(e.path); + e.resume(); + }); + + extract.on('end', () => { + paths = paths.sort(); + assert.deepStrictEqual( + [ + 'apples/.ignore', 'apples/index.js', 'apples/lib/', + 'apples/lib/a-file.js', 'apples/test/', 'apples/test/test.js' + ], + paths, 'ignored files in globs in ignore files'); + + console.log(paths); + done(); + }); + }); + + + it('packs a directory honoring an ignore string', (done) => { + const nodeTar = require('tar'); + + const fixtureDir = path.join(__dirname, '..', '..', 'fixtures', 'project'); + + const tar = pack({[fixtureDir]: '/apples'}, {ignores: ['**/test']}); + + let paths: string[] = []; + + const extract = tar.pipe(nodeTar.t()); + extract.on('entry', (e: Readable&{path: string}) => { + paths.push(e.path); + e.resume(); + }); + + extract.on('end', () => { + paths = paths.sort(); + assert.deepStrictEqual( + [ + 'apples/.ignore', 'apples/index.js', 'apples/lib/', + 'apples/lib/a-file.js' + ], + paths, 'ignored test files with **/test glob'); + + console.log(paths); + done(); + }); + }); +}); diff --git a/test/walker.ts b/test/walker.ts index eb4aa04..67c405c 100644 --- a/test/walker.ts +++ b/test/walker.ts @@ -30,4 +30,29 @@ describe('ignore file walker', () => { result.indexOf('/taco.yaml') === -1, 'should have honored ignore file and removed taco.yaml'); }); + + + it('ignores', async () => { + let result = + await walker.walk( + fixturePath, + {return_object: false, find_links: false, ignores: ['**/test']}) as + string[]; + result = result.map((path) => path.replace(fixturePath, '')); + /* + [ '/.ignore', + '/index.js', + '/test', + '/lib', + '/test/taco.yaml', + '/test/test.js', + '/lib/a-file.js' ] + */ + + console.log(result); + + assert.ok( + result.indexOf('/test/test.js') === -1, + 'should have honored ignore string and removed test files'); + }); }); \ No newline at end of file