diff --git a/.cicd/README.md b/.cicd/README.md new file mode 100644 index 00000000000..93ad073ae53 --- /dev/null +++ b/.cicd/README.md @@ -0,0 +1,105 @@ +# eosio +The [eosio](https://buildkite.com/EOSIO/eosio) and [eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned) pipelines are the primary pipelines for the [eos](https://github.com/EOSIO/eos) repository, running with specific or default versions of our dependencies, respectively. Both run against every commit to a base branch or pull request, along with the [eosio-code-coverage](https://buildkite.com/EOSIO/eosio-code-coverage) pipeline. + +The [eosio](https://buildkite.com/EOSIO/eosio) pipeline further triggers the [eosio-sync-from-genesis](https://buildkite.com/EOSIO/eosio-sync-from-genesis) and [eosio-resume-from-state](https://buildkite.com/EOSIO/eosio-resume-from-state) pipelines on each build, and the the [eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) pipeline on merge commits. Each of these pipelines are described in more detail below and in their respective READMEs. + +
+See More + +## Index +1. [Configuration](README.md#configuration) + 1. [Variables](README.md#variables) + 1. [Examples](README.md#examples) +1. [Pipelines](README.md#pipelines) +1. [See Also](README.md#see-also) + +## Configuration +Most EOSIO pipelines are run any time you push a commit or tag to an open pull request in [eos](https://github.com/EOSIO/eos), any time you merge a pull request, and nightly. The [eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) pipeline only runs when you merge a pull request because it takes so long. Long-running tests are also run in the [eosio](https://buildkite.com/EOSIO/eosio) nightly builds, which have `RUN_ALL_TESTS='true'` set. + +### Variables +Most pipelines in the organization have several environment variables that can be used to configure how the pipeline runs. These environment variables can be specified when manually triggering a build via the Buildkite UI. + +Configure which platforms are run: +```bash +SKIP_LINUX='true|false' # skip all steps on Linux distros +SKIP_MAC='true|false' # skip all steps on Mac hardware +``` +These will override more specific operating system declarations, and primarily exist to disable one of our two buildfleets should one be sick or the finite macOS agents are congested. + +Configure which operating systems are built, tested, and packaged: +```bash +RUN_ALL_TESTS='true' # run all tests in the current build (including LRTs, overridden by SKIP* variables) +SKIP_AMAZON_LINUX_2='true|false' # skip all steps for Amazon Linux 2 +SKIP_CENTOS_7_7='true|false' # skip all steps for Centos 7.7 +SKIP_CENTOS_8='true|false' # skip all steps for Centos 8 +SKIP_MACOS_10_14='true|false' # skip all steps for MacOS 10.14 +SKIP_MACOS_10_15='true|false' # skip all steps for MacOS 10.15 +SKIP_MACOS_11='true|false' # skip all steps for MacOS 11 +SKIP_UBUNTU_16_04='true|false' # skip all steps for Ubuntu 16.04 +SKIP_UBUNTU_18_04='true|false' # skip all steps for Ubuntu 18.04 +SKIP_UBUNTU_20_04='true|false' # skip all steps for Ubuntu 20.04 +``` + +Configure which steps are executed for each operating system: +```bash +SKIP_BUILD='true|false' # skip all build steps +SKIP_UNIT_TESTS='true|false' # skip all unit tests +SKIP_WASM_SPEC_TESTS='true|false' # skip all wasm spec tests +SKIP_SERIAL_TESTS='true|false' # skip all integration tests +SKIP_LONG_RUNNING_TESTS='true|false' # skip all long running tests +SKIP_MULTIVERSION_TEST='true|false' # skip all multiversion tests +SKIP_SYNC_TESTS='true|false' # skip all sync tests +SKIP_PACKAGE_BUILDER='true|false' # skip all packaging steps +``` + +Configure how the steps are executed: +```bash +PINNED='true|false' # use specific versions of dependencies instead of whatever version is provided by default on a given platform +TIMEOUT='##' # set timeout in minutes for all steps +``` + +### Examples +Build and test on Linux only: +```bash +SKIP_MAC='true' +``` + +Build and test on MacOS only: +```bash +SKIP_LINUX='true' +``` + +Skip all tests: +```bash +SKIP_UNIT_TESTS='true' +SKIP_WASM_SPEC_TESTS='true' +SKIP_SERIAL_TESTS='true' +SKIP_LONG_RUNNING_TESTS='true' +SKIP_MULTIVERSION_TEST='true' +SKIP_SYNC_TESTS='true' +``` + +## Pipelines +There are several eosio pipelines that exist and are triggered by pull requests, pipelines, or schedules: + +Pipeline | Details +---|--- +[eosio](https://buildkite.com/EOSIO/eosio) | [eos](https://github.com/EOSIO/eos) build, tests, and packaging with pinned dependencies; runs on every pull request and base branch commit, and nightly +[eosio-base-images](https://buildkite.com/EOSIO/eosio-base-images) | pack EOSIO dependencies into docker and Anka base-images nightly +[eosio-big-sur-beta](https://buildkite.com/EOSIO/eosio-big-sur-beta) | build and test [eos](https://github.com/EOSIO/eos) on macOS 11 "Big Sur" weekly +[eosio-build-scripts](https://buildkite.com/EOSIO/eosio-build-scripts) | run [eos](https://github.com/EOSIO/eos) build scripts nightly on empty operating systems +[eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned) | [eos](https://github.com/EOSIO/eos) build and tests with platform-provided dependencies; runs on every pull request and base branch commit, and nightly +[eosio-code-coverage](https://buildkite.com/EOSIO/eosio-code-coverage) | assess [eos](https://github.com/EOSIO/eos) unit test coverage; runs on every pull request and base branch commit +[eosio-debug-build](https://buildkite.com/EOSIO/eosio-debug-build) | perform a debug build for [eos](https://github.com/EOSIO/eos) on every pull request and base branch commit +[eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) | runs tests that need more time on merge commits +[eosio-resume-from-state](https://buildkite.com/EOSIO/eosio-resume-from-state) | loads the current version of `nodeos` from state files generated by specific previous versions of `nodeos` in each [eosio](https://buildkite.com/EOSIO/eosio) build ([Documentation](https://github.com/EOSIO/auto-eks-sync-nodes/blob/master/pipelines/eosio-resume-from-state/README.md)) +[eosio-sync-from-genesis](https://buildkite.com/EOSIO/eosio-sync-from-genesis) | sync the current version of `nodeos` past genesis from peers on common public chains as a smoke test, for each [eosio](https://buildkite.com/EOSIO/eosio) build + +## See Also +- Buildkite + - [DevDocs](https://github.com/EOSIO/devdocs/wiki/Buildkite) + - [eosio-resume-from-state Documentation](https://github.com/EOSIO/auto-eks-sync-nodes/blob/master/pipelines/eosio-resume-from-state/README.md) + - [Run Your First Build](https://buildkite.com/docs/tutorials/getting-started#run-your-first-build) +- [#help-automation](https://blockone.slack.com/archives/CMTAZ9L4D) Slack Channel + +
diff --git a/.cicd/base-images.yml b/.cicd/base-images.yml index cf142664186..717193f4fe1 100644 --- a/.cicd/base-images.yml +++ b/.cicd/base-images.yml @@ -45,6 +45,29 @@ steps: cd: ~ agents: "queue=mac-anka-node-fleet" timeout: 180 + + - label: ":darwin: macOS 10.15 - Base Image Pinned" + command: + - "git clone git@github.com:EOSIO/eos.git eos && cd eos && git checkout -f $BUILDKITE_BRANCH" + - "cd eos && ./.cicd/platforms/pinned/macos-10.15-pinned.sh" + plugins: + - EOSIO/anka#v0.6.1: + debug: true + vm-name: "10.15.5_6C_14G_80G" + no-volume: true + always-pull: true + wait-network: true + pre-execute-sleep: 5 + pre-execute-ping-sleep: github.com + vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent" + failover-registries: + - "registry_1" + - "registry_2" + inherit-environment-vars: true + - EOSIO/skip-checkout#v0.1.1: + cd: ~ + agents: "queue=mac-anka-node-fleet" + timeout: 180 - label: ":ubuntu: Ubuntu 16.04 - Base Image Pinned" command: @@ -112,6 +135,29 @@ steps: cd: ~ agents: "queue=mac-anka-node-fleet" timeout: 180 + + - label: ":darwin: macOS 10.15 - Base Image Unpinned" + command: + - "git clone git@github.com:EOSIO/eos.git eos && cd eos && git checkout -f $BUILDKITE_BRANCH" + - "cd eos && ./.cicd/platforms/unpinned/macos-10.15-unpinned.sh" + plugins: + - EOSIO/anka#v0.6.1: + debug: true + vm-name: "10.15.5_6C_14G_80G" + no-volume: true + always-pull: true + wait-network: true + pre-execute-sleep: 5 + pre-execute-ping-sleep: github.com + vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent" + failover-registries: + - "registry_1" + - "registry_2" + inherit-environment-vars: true + - EOSIO/skip-checkout#v0.1.1: + cd: ~ + agents: "queue=mac-anka-node-fleet" + timeout: 180 - label: ":ubuntu: Ubuntu 18.04 - Base Image Unpinned" command: diff --git a/.cicd/build-scripts.yml b/.cicd/build-scripts.yml index b3018066b5d..2d09492fa19 100644 --- a/.cicd/build-scripts.yml +++ b/.cicd/build-scripts.yml @@ -21,7 +21,18 @@ steps: command: - "./scripts/eosio_build.sh -P -y" timeout: 180 - + + - label: ":centos: CentOS 8 - Build Pinned" + plugins: + - docker#v3.3.0: + image: "centos:8" + always-pull: true + agents: + queue: "automation-eks-eos-builder-fleet" + command: + - "./scripts/eosio_build.sh -P -y" + timeout: 180 + - label: ":darwin: macOS 10.14 - Build Pinned" env: REPO: "git@github.com:EOSIO/eos.git" @@ -29,7 +40,7 @@ steps: TEMPLATE_TAG: "clean::cicd::git-ssh::nas::brew::buildkite-agent" agents: "queue=mac-anka-large-node-fleet" command: - - "git clone git@github.com:EOSIO/eos.git eos && cd eos && git checkout -f $BUILDKITE_BRANCH && git submodule update --init --recursive" + - "git clone git@github.com:EOSIO/eos.git eos && cd eos && git checkout -f $BUILDKITE_BRANCH && git submodule update --init --recursive" - "cd eos && ./scripts/eosio_build.sh -P -y" plugins: - EOSIO/anka#v0.6.1: @@ -50,6 +61,36 @@ steps: - EOSIO/skip-checkout#v0.1.1: cd: ~ timeout: 180 + + + - label: ":darwin: macOS 10.15 - Build Pinned" + env: + REPO: "git@github.com:EOSIO/eos.git" + TEMPLATE: "10.15.5_6C_14G_80G" + TEMPLATE_TAG: "clean::cicd::git-ssh::nas::brew::buildkite-agent" + agents: "queue=mac-anka-large-node-fleet" + command: + - "git clone git@github.com:EOSIO/eos.git eos && cd eos && git checkout -f $BUILDKITE_BRANCH && git submodule update --init --recursive" + - "cd eos && ./scripts/eosio_build.sh -P -y" + plugins: + - EOSIO/anka#v0.6.1: + debug: true + vm-name: "10.15.5_6C_14G_80G" + no-volume: true + modify-cpu: 12 + modify-ram: 24 + always-pull: true + wait-network: true + pre-execute-sleep: 5 + pre-execute-ping-sleep: github.com + vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent" + failover-registries: + - "registry_1" + - "registry_2" + inherit-environment-vars: true + - EOSIO/skip-checkout#v0.1.1: + cd: ~ + timeout: 180 - label: ":ubuntu: Ubuntu 16.04 - Build Pinned" plugins: @@ -75,6 +116,21 @@ steps: - "./scripts/eosio_build.sh -P -y" timeout: 180 + - label: ":ubuntu: Ubuntu 20.04 - Build Pinned" + env: + DEBIAN_FRONTEND: "noninteractive" + plugins: + - docker#v3.3.0: + image: "ubuntu:20.04" + always-pull: true + agents: + queue: "automation-eks-eos-builder-fleet" + command: + - "ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime" + - "apt update && apt upgrade -y && apt install -y git" + - "./scripts/eosio_build.sh -P -y" + timeout: 180 + - label: ":aws: Amazon_Linux 2 - Build UnPinned" plugins: - docker#v3.3.0: @@ -96,6 +152,17 @@ steps: command: - "./scripts/eosio_build.sh -y" timeout: 180 + + - label: ":centos: CentOS 8 - Build UnPinned" + plugins: + - docker#v3.3.0: + image: "centos:8" + always-pull: true + agents: + queue: "automation-eks-eos-builder-fleet" + command: + - "./scripts/eosio_build.sh -y" + timeout: 180 - label: ":darwin: macOS 10.14 - Build UnPinned" env: @@ -125,6 +192,35 @@ steps: - EOSIO/skip-checkout#v0.1.1: cd: ~ timeout: 180 + + - label: ":darwin: macOS 10.15 - Build UnPinned" + env: + REPO: "git@github.com:EOSIO/eos.git" + TEMPLATE: "10.15.5_6C_14G_80G" + TEMPLATE_TAG: "clean::cicd::git-ssh::nas::brew::buildkite-agent" + agents: "queue=mac-anka-large-node-fleet" + command: + - "git clone git@github.com:EOSIO/eos.git eos && cd eos && git checkout -f $BUILDKITE_BRANCH && git submodule update --init --recursive" + - "cd eos && ./scripts/eosio_build.sh -y" + plugins: + - EOSIO/anka#v0.6.1: + debug: true + vm-name: "10.15.5_6C_14G_80G" + no-volume: true + modify-cpu: 12 + modify-ram: 24 + always-pull: true + wait-network: true + pre-execute-sleep: 5 + pre-execute-ping-sleep: github.com + vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent" + failover-registries: + - "registry_1" + - "registry_2" + inherit-environment-vars: true + - EOSIO/skip-checkout#v0.1.1: + cd: ~ + timeout: 180 - label: ":ubuntu: Ubuntu 18.04 - Build UnPinned" plugins: @@ -136,4 +232,19 @@ steps: command: - "apt update && apt upgrade -y && apt install -y git" - "./scripts/eosio_build.sh -y" - timeout: 180 \ No newline at end of file + timeout: 180 + + - label: ":ubuntu: Ubuntu 20.04 - Build UnPinned" + env: + DEBIAN_FRONTEND: "noninteractive" + plugins: + - docker#v3.3.0: + image: "ubuntu:20.04" + always-pull: true + agents: + queue: "automation-eks-eos-builder-fleet" + command: + - "ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime" + - "apt update && apt upgrade -y && apt install -y git g++" + - "./scripts/eosio_build.sh -y" + timeout: 180 diff --git a/.cicd/build.sh b/.cicd/build.sh index 5ffcff277fb..1989f507851 100755 --- a/.cicd/build.sh +++ b/.cicd/build.sh @@ -4,7 +4,7 @@ set -eo pipefail . ./.cicd/helpers/general.sh mkdir -p "$BUILD_DIR" [[ -z "$DCMAKE_BUILD_TYPE" ]] && export DCMAKE_BUILD_TYPE='Release' -CMAKE_EXTRAS="-DCMAKE_BUILD_TYPE=\"$DCMAKE_BUILD_TYPE\" -DENABLE_MULTIVERSION_PROTOCOL_TEST=\"true\" -DBUILD_MONGO_DB_PLUGIN=\"true\"" +CMAKE_EXTRAS="-DCMAKE_BUILD_TYPE=\"$DCMAKE_BUILD_TYPE\" -DENABLE_MULTIVERSION_PROTOCOL_TEST=\"true\"" if [[ "$(uname)" == 'Darwin' && "$FORCE_LINUX" != 'true' ]]; then # You can't use chained commands in execute if [[ "$GITHUB_ACTIONS" == 'true' ]]; then @@ -12,6 +12,7 @@ if [[ "$(uname)" == 'Darwin' && "$FORCE_LINUX" != 'true' ]]; then fi [[ ! "$PINNED" == 'false' ]] && CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_TOOLCHAIN_FILE=\"$HELPERS_DIR/clang.make\"" cd "$BUILD_DIR" + [[ "$CI" == 'true' ]] && source ~/.bash_profile # Make sure node is available for ship_test echo '+++ :hammer_and_wrench: Building EOSIO' CMAKE_COMMAND="cmake $CMAKE_EXTRAS .." echo "$ $CMAKE_COMMAND" @@ -29,11 +30,14 @@ else # Linux if [[ "$IMAGE_TAG" == 'amazon_linux-2-unpinned' ]]; then CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_COMPILER=\"clang++\" -DCMAKE_C_COMPILER=\"clang\"" elif [[ "$IMAGE_TAG" == 'centos-7.7-unpinned' ]]; then - PRE_COMMANDS="$PRE_COMMANDS && source \"/opt/rh/devtoolset-8/enable\" && source \"/opt/rh/rh-python36/enable\"" + PRE_COMMANDS="$PRE_COMMANDS && source /opt/rh/devtoolset-8/enable" CMAKE_EXTRAS="$CMAKE_EXTRAS -DLLVM_DIR=\"/opt/rh/llvm-toolset-7.0/root/usr/lib64/cmake/llvm\"" elif [[ "$IMAGE_TAG" == 'ubuntu-18.04-unpinned' ]]; then CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_COMPILER=\"clang++-7\" -DCMAKE_C_COMPILER=\"clang-7\" -DLLVM_DIR=\"/usr/lib/llvm-7/lib/cmake/llvm\"" fi + if [[ "$IMAGE_TAG" == centos-7.* ]]; then + PRE_COMMANDS="$PRE_COMMANDS && source /opt/rh/rh-python36/enable" + fi CMAKE_COMMAND="cmake \$CMAKE_EXTRAS .." MAKE_COMMAND="make -j $JOBS" BUILD_COMMANDS="echo \"+++ :hammer_and_wrench: Building EOSIO\" && echo \"$ $CMAKE_COMMAND\" && eval $CMAKE_COMMAND && echo \"$ $MAKE_COMMAND\" && eval $MAKE_COMMAND" diff --git a/.cicd/create-docker-from-binary.sh b/.cicd/create-docker-from-binary.sh index aa58f3e1b38..42bbfc47569 100755 --- a/.cicd/create-docker-from-binary.sh +++ b/.cicd/create-docker-from-binary.sh @@ -1,54 +1,65 @@ #!/bin/bash - +echo '--- :evergreen_tree: Configuring Environment' set -euo pipefail - +. ./.cicd/helpers/general.sh buildkite-agent artifact download '*.deb' --step ':ubuntu: Ubuntu 18.04 - Package Builder' . -echo ":done: download successful" - -SANITIZED_BRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's.^/..' | sed 's/[:/]/_/g') -SANITIZED_TAG=$(echo "$BUILDKITE_TAG" | sed 's.^/..' | tr '/' '_') -echo "$SANITIZED_BRANCH" -echo "$SANITIZED_TAG" - -# do docker build -echo ":docker::build: Building image..." -DOCKERHUB_REGISTRY="docker.io/eosio/eosio" - -BUILD_TAG=${BUILDKITE_BUILD_NUMBER:-latest} -DOCKER_BUILD_GEN="docker build -t eosio_image:$BUILD_TAG -f ./docker/dockerfile ." -echo "$ $DOCKER_BUILD_GEN" -eval $DOCKER_BUILD_GEN - -#tag and push on each destination AWS & DOCKERHUB - -EOSIO_REGS=("$EOSIO_REGISTRY" "$DOCKERHUB_REGISTRY") -for REG in ${EOSIO_REGS[@]}; do - DOCKER_TAG_COMMIT="docker tag eosio_image:$BUILD_TAG $REG:$BUILDKITE_COMMIT" - DOCKER_TAG_BRANCH="docker tag eosio_image:$BUILD_TAG $REG:$SANITIZED_BRANCH" - echo -e "$ Tagging Images: \n$DOCKER_TAG_COMMIT \n$DOCKER_TAG_BRANCH" - eval $DOCKER_TAG_COMMIT +SANITIZED_BRANCH="$(sanitize "$BUILDKITE_BRANCH")" +echo "Branch '$BUILDKITE_BRANCH' sanitized as '$SANITIZED_BRANCH'." +SANITIZED_TAG="$(sanitize "$BUILDKITE_TAG")" +[[ -z "$SANITIZED_TAG" ]] || echo "Branch '$BUILDKITE_TAG' sanitized as '$SANITIZED_TAG'." +# docker build +echo "+++ :docker: Build Docker Container" +DOCKERHUB_REGISTRY='docker.io/eosio/eosio' +IMAGE="${DOCKERHUB_REGISTRY}:${BUILDKITE_COMMIT:-latest}" +DOCKER_BUILD="docker build -t '$IMAGE' -f ./docker/dockerfile ." +echo "$ $DOCKER_BUILD" +eval $DOCKER_BUILD +# docker tag +echo '--- :label: Tag Container' +REGISTRIES=("$EOSIO_REGISTRY" "$DOCKERHUB_REGISTRY") +for REG in ${REGISTRIES[@]}; do + DOCKER_TAG_BRANCH="docker tag '$IMAGE' '$REG:$SANITIZED_BRANCH'" + echo "$ $DOCKER_TAG_BRANCH" eval $DOCKER_TAG_BRANCH - DOCKER_PUSH_COMMIT="docker push $REG:$BUILDKITE_COMMIT" - DOCKER_PUSH_BRANCH="docker push $REG:$SANITIZED_BRANCH" - echo -e "$ Pushing Images: \n$DOCKER_PUSH_COMMIT \n$DOCKER_PUSH_BRANCH" - eval $DOCKER_PUSH_COMMIT - eval $DOCKER_PUSH_BRANCH - CLEAN_IMAGE_COMMIT="docker rmi $REG:$BUILDKITE_COMMIT || :" - CLEAN_IMAGE_BRANCH="docker rmi $REG:$SANITIZED_BRANCH || :" - echo -e "Cleaning Up: \n$CLEAN_IMAGE_COMMIT \n$CLEAN_IMAGE_BRANCH$" - eval $CLEAN_IMAGE_COMMIT - eval $CLEAN_IMAGE_BRANCH - if [[ ! -z "$SANITIZED_TAG" ]]; then - DOCKER_TAG="docker tag eosio_image:$BUILD_TAG $REG:$SANITIZED_TAG" - DOCKER_PUSH_TAG="docker push $REG:$SANITIZED_TAG" - DOCKER_REM="docker rmi $REG:$SANITIZED_TAG || :" - echo -e "$ \n Tagging Image: \n$DOCKER_TAG \n Cleaning Up: \n$DOCKER_REM" + DOCKER_TAG_COMMIT="docker tag '$IMAGE' '$REG:$BUILDKITE_COMMIT'" + echo "$ $DOCKER_TAG_COMMIT" + eval $DOCKER_TAG_COMMIT + if [[ ! -z "$SANITIZED_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then + DOCKER_TAG="docker tag '$IMAGE' '$REG:$SANITIZED_TAG'" + echo "$ $DOCKER_TAG" eval $DOCKER_TAG + fi +done +# docker push +echo '--- :arrow_up: Push Container' +for REG in ${REGISTRIES[@]}; do + DOCKER_PUSH_BRANCH="docker push '$REG:$SANITIZED_BRANCH'" + echo "$ $DOCKER_PUSH_BRANCH" + eval $DOCKER_PUSH_BRANCH + DOCKER_PUSH_COMMIT="docker push '$REG:$BUILDKITE_COMMIT'" + echo "$ $DOCKER_PUSH_COMMIT" + eval $DOCKER_PUSH_COMMIT + if [[ ! -z "$SANITIZED_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then + DOCKER_PUSH_TAG="docker push '$REG:$SANITIZED_TAG'" + echo "$ $DOCKER_PUSH_TAG" eval $DOCKER_PUSH_TAG - eval $DOCKER_REM fi done - -DOCKER_GEN="docker rmi eosio_image:$BUILD_TAG || :" -echo "Clean up base image" -eval $DOCKER_GEN \ No newline at end of file +# docker rmi +echo '--- :put_litter_in_its_place: Cleanup' +for REG in ${REGISTRIES[@]}; do + CLEAN_IMAGE_BRANCH="docker rmi '$REG:$SANITIZED_BRANCH' || :" + echo "$ $CLEAN_IMAGE_BRANCH" + eval $CLEAN_IMAGE_BRANCH + CLEAN_IMAGE_COMMIT="docker rmi '$REG:$BUILDKITE_COMMIT' || :" + echo "$ $CLEAN_IMAGE_COMMIT" + eval $CLEAN_IMAGE_COMMIT + if [[ ! -z "$SANITIZED_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then + DOCKER_RMI="docker rmi '$REG:$SANITIZED_TAG' || :" + echo "$ $DOCKER_RMI" + eval $DOCKER_RMI + fi +done +DOCKER_RMI="docker rmi '$IMAGE' || :" +echo "$ $DOCKER_RMI" +eval $DOCKER_RMI diff --git a/.cicd/docker-tag.sh b/.cicd/docker-tag.sh index 85142e60b52..33211ac88d8 100755 --- a/.cicd/docker-tag.sh +++ b/.cicd/docker-tag.sh @@ -3,8 +3,10 @@ set -eo pipefail echo '--- :evergreen_tree: Configuring Environment' . ./.cicd/helpers/general.sh PREFIX='base-ubuntu-18.04' -SANITIZED_BRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's.^/..' | sed 's/[:/]/_/g') -SANITIZED_TAG=$(echo "$BUILDKITE_TAG" | sed 's.^/..' | tr '/' '_') +SANITIZED_BRANCH="$(sanitize "$BUILDKITE_BRANCH")" +echo "Branch '$BUILDKITE_BRANCH' sanitized as '$SANITIZED_BRANCH'." +SANITIZED_TAG="$(sanitize "$BUILDKITE_TAG")" +[[ -z "$SANITIZED_TAG" ]] || echo "Branch '$BUILDKITE_TAG' sanitized as '$SANITIZED_TAG'." echo '$ echo ${#CONTRACT_REGISTRIES[*]} # array length' echo ${#CONTRACT_REGISTRIES[*]} echo '$ echo ${CONTRACT_REGISTRIES[*]} # array' @@ -23,7 +25,7 @@ for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do DOCKER_TAG_COMMAND="docker tag '$IMAGE' '$REGISTRY:$PREFIX-$SANITIZED_BRANCH'" echo "$ $DOCKER_TAG_COMMAND" eval $DOCKER_TAG_COMMAND - if [[ ! -z "$BUILDKITE_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then + if [[ ! -z "$SANITIZED_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then DOCKER_TAG_COMMAND="docker tag '$IMAGE' '$REGISTRY:$PREFIX-$SANITIZED_TAG'" echo "$ $DOCKER_TAG_COMMAND" eval $DOCKER_TAG_COMMAND @@ -38,7 +40,7 @@ for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do DOCKER_PUSH_COMMAND="docker push '$REGISTRY:$PREFIX-$SANITIZED_BRANCH'" echo "$ $DOCKER_PUSH_COMMAND" eval $DOCKER_PUSH_COMMAND - if [[ ! -z "$BUILDKITE_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then + if [[ ! -z "$SANITIZED_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then DOCKER_PUSH_COMMAND="docker push '$REGISTRY:$PREFIX-$SANITIZED_TAG'" echo "$ $DOCKER_PUSH_COMMAND" eval $DOCKER_PUSH_COMMAND @@ -56,7 +58,7 @@ for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$BUILDKITE_COMMIT' || :" echo "$ $DOCKER_RMI_COMMAND" eval $DOCKER_RMI_COMMAND - if [[ ! -z "$BUILDKITE_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then + if [[ ! -z "$SANITIZED_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$SANITIZED_TAG' || :" echo "$ $DOCKER_RMI_COMMAND" eval $DOCKER_RMI_COMMAND diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 71f96466de3..511779fb8f4 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -2,18 +2,22 @@ set -eo pipefail # environment . ./.cicd/helpers/general.sh -export MOJAVE_ANKA_TAG_BASE=${MOJAVE_ANKA_TAG_BASE:-'clean::cicd::git-ssh::nas::brew::buildkite-agent'} -export MOJAVE_ANKA_TEMPLATE_NAME=${MOJAVE_ANKA_TEMPLATE_NAME:-'10.14.6_6C_14G_80G'} export PLATFORMS_JSON_ARRAY='[]' +[[ -z "$ROUNDS" ]] && export ROUNDS='1' BUILDKITE_BUILD_AGENT_QUEUE='automation-eks-eos-builder-fleet' BUILDKITE_TEST_AGENT_QUEUE='automation-eks-eos-tester-fleet' -[[ -z "$ROUNDS" ]] && export ROUNDS='1' +# attach pipeline documentation +export DOCS_URL="https://github.com/EOSIO/eos/blob/${BUILDKITE_COMMIT:-master}/.cicd/README.md" +export RETRY="$(buildkite-agent meta-data get pipeline-upload-retries --default '0')" +if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then + echo "This documentation is also available on [GitHub]($DOCS_URL)." | buildkite-agent annotate --append --style 'info' --context 'documentation' + cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' +fi # Determine if it's a forked PR and make sure to add git fetch so we don't have to git clone the forked repo's url if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then PR_ID=$(echo $BUILDKITE_BRANCH | cut -d/ -f2) export GIT_FETCH="git fetch -v --prune origin refs/pull/$PR_ID/head &&" fi -[[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio-debug-build' ]] && export SKIP_UBUNTU_18_04='false' # Determine which dockerfiles/scripts to use for the pipeline. if [[ $PINNED == false ]]; then export PLATFORM_TYPE="unpinned" @@ -32,6 +36,11 @@ for FILE in $(ls "$CICD_DIR/platforms/$PLATFORM_TYPE"); do export FILE_NAME="$(echo "$FILE" | awk '{split($0,a,/\.(d|s)/); print a[1] }')" # macos-10.14 # ubuntu-16.04 + # skip Mojave if it's anything but the post-merge build + if [[ "$FILE_NAME" =~ 'macos-10.14' && "$SKIP_MACOS_10_14" != 'false' && "$RUN_ALL_TESTS" != 'true' && ( "$BUILDKITE_SOURCE" != 'webhook' || "$BUILDKITE_PULL_REQUEST" != 'false' || ! "$BUILDKITE_MESSAGE" =~ 'Merge pull request' ) ]]; then + export SKIP_MACOS_10_14='true' + continue + fi export PLATFORM_NAME="$(echo $FILE_NAME | cut -d- -f1 | sed 's/os/OS/g')" # macOS # ubuntu @@ -58,6 +67,16 @@ for FILE in $(ls "$CICD_DIR/platforms/$PLATFORM_TYPE"); do [[ $FILE_NAME =~ 'macos' ]] && export ICON=':darwin:' . "$HELPERS_DIR/file-hash.sh" "$CICD_DIR/platforms/$PLATFORM_TYPE/$FILE" # returns HASHED_IMAGE_TAG, etc export PLATFORM_SKIP_VAR="SKIP_${PLATFORM_NAME_UPCASE}_${VERSION_MAJOR}${VERSION_MINOR}" + # Anka Template and Tags + export ANKA_TAG_BASE='clean::cicd::git-ssh::nas::brew::buildkite-agent' + if [[ $FILE_NAME =~ 'macos-10.14' ]]; then + export ANKA_TEMPLATE_NAME='10.14.6_6C_14G_80G' + elif [[ $FILE_NAME =~ 'macos-10.15' ]]; then + export ANKA_TEMPLATE_NAME='10.15.5_6C_14G_80G' + else # Linux + export ANKA_TAG_BASE='' + export ANKA_TEMPLATE_NAME='' + fi export PLATFORMS_JSON_ARRAY=$(echo $PLATFORMS_JSON_ARRAY | jq -c '. += [{ "FILE_NAME": env.FILE_NAME, "PLATFORM_NAME": env.PLATFORM_NAME, @@ -69,7 +88,9 @@ for FILE in $(ls "$CICD_DIR/platforms/$PLATFORM_TYPE"); do "PLATFORM_NAME_FULL": env.PLATFORM_NAME_FULL, "DOCKERHUB_FULL_TAG": env.FULL_TAG, "HASHED_IMAGE_TAG": env.HASHED_IMAGE_TAG, - "ICON": env.ICON + "ICON": env.ICON, + "ANKA_TAG_BASE": env.ANKA_TAG_BASE, + "ANKA_TEMPLATE_NAME": env.ANKA_TEMPLATE_NAME }]') done # set build_source whether triggered or not @@ -91,6 +112,7 @@ oIFS="$IFS" IFS=$'' nIFS=$IFS # fix array splitting (\n won't work) # start with a wait step +echo 'steps:' echo ' - wait' echo '' # build steps @@ -129,8 +151,8 @@ EOF - EOSIO/anka#v0.6.1: no-volume: true inherit-environment-vars: true - vm-name: ${MOJAVE_ANKA_TEMPLATE_NAME} - vm-registry-tag: "${MOJAVE_ANKA_TAG_BASE}::$(echo "$PLATFORM_JSON" | jq -r .HASHED_IMAGE_TAG)" + vm-name: $(echo "$PLATFORM_JSON" | jq -r .ANKA_TEMPLATE_NAME) + vm-registry-tag: $(echo "$PLATFORM_JSON" | jq -r .ANKA_TAG_BASE)::$(echo "$PLATFORM_JSON" | jq -r .HASHED_IMAGE_TAG) modify-cpu: 12 modify-ram: 24 always-pull: true @@ -142,15 +164,15 @@ EOF - 'registry_1' - 'registry_2' pre-commands: - - "git clone git@github.com:EOSIO/mac-anka-fleet.git && cd mac-anka-fleet && . ./ensure-tag.bash -u 12 -r 25G -a '-n'" + - "rm -rf mac-anka-fleet; git clone git@github.com:EOSIO/mac-anka-fleet.git && cd mac-anka-fleet && . ./ensure-tag.bash -u 12 -r 25G -a '-n'" - EOSIO/skip-checkout#v0.1.1: cd: ~ env: DCMAKE_BUILD_TYPE: $DCMAKE_BUILD_TYPE REPO: ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO} REPO_COMMIT: $BUILDKITE_COMMIT - TEMPLATE: $MOJAVE_ANKA_TEMPLATE_NAME - TEMPLATE_TAG: $MOJAVE_ANKA_TAG_BASE + TEMPLATE: $(echo "$PLATFORM_JSON" | jq -r .ANKA_TEMPLATE_NAME) + TEMPLATE_TAG: $(echo "$PLATFORM_JSON" | jq -r .ANKA_TAG_BASE) IMAGE_TAG: $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME) PLATFORM_TYPE: $PLATFORM_TYPE TAG_COMMANDS: "git clone ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO} eos && cd eos && $GIT_FETCH git checkout -f \$BUILDKITE_COMMIT && git submodule update --init --recursive && export IMAGE_TAG=$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME) && export PLATFORM_TYPE=$PLATFORM_TYPE && . ./.cicd/platforms/$PLATFORM_TYPE/$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME).sh && cd ~/eos && cd .. && rm -rf eos" @@ -215,8 +237,8 @@ EOF - EOSIO/anka#v0.6.1: no-volume: true inherit-environment-vars: true - vm-name: ${MOJAVE_ANKA_TEMPLATE_NAME} - vm-registry-tag: "${MOJAVE_ANKA_TAG_BASE}::$(echo "$PLATFORM_JSON" | jq -r .HASHED_IMAGE_TAG)" + vm-name: $(echo "$PLATFORM_JSON" | jq -r .ANKA_TEMPLATE_NAME) + vm-registry-tag: $(echo "$PLATFORM_JSON" | jq -r .ANKA_TAG_BASE)::$(echo "$PLATFORM_JSON" | jq -r .HASHED_IMAGE_TAG) always-pull: true debug: true wait-network: true @@ -273,8 +295,8 @@ EOF - EOSIO/anka#v0.6.1: no-volume: true inherit-environment-vars: true - vm-name: ${MOJAVE_ANKA_TEMPLATE_NAME} - vm-registry-tag: "${MOJAVE_ANKA_TAG_BASE}::$(echo "$PLATFORM_JSON" | jq -r .HASHED_IMAGE_TAG)" + vm-name: $(echo "$PLATFORM_JSON" | jq -r .ANKA_TEMPLATE_NAME) + vm-registry-tag: $(echo "$PLATFORM_JSON" | jq -r .ANKA_TAG_BASE)::$(echo "$PLATFORM_JSON" | jq -r .HASHED_IMAGE_TAG) always-pull: true debug: true wait-network: true @@ -331,8 +353,8 @@ EOF - EOSIO/anka#v0.6.1: no-volume: true inherit-environment-vars: true - vm-name: ${MOJAVE_ANKA_TEMPLATE_NAME} - vm-registry-tag: "${MOJAVE_ANKA_TAG_BASE}::$(echo "$PLATFORM_JSON" | jq -r .HASHED_IMAGE_TAG)" + vm-name: $(echo "$PLATFORM_JSON" | jq -r .ANKA_TEMPLATE_NAME) + vm-registry-tag: $(echo "$PLATFORM_JSON" | jq -r .ANKA_TAG_BASE)::$(echo "$PLATFORM_JSON" | jq -r .HASHED_IMAGE_TAG) always-pull: true debug: true wait-network: true @@ -391,8 +413,8 @@ EOF - EOSIO/anka#v0.6.1: no-volume: true inherit-environment-vars: true - vm-name: ${MOJAVE_ANKA_TEMPLATE_NAME} - vm-registry-tag: "${MOJAVE_ANKA_TAG_BASE}::$(echo "$PLATFORM_JSON" | jq -r .HASHED_IMAGE_TAG)" + vm-name: $(echo "$PLATFORM_JSON" | jq -r .ANKA_TEMPLATE_NAME) + vm-registry-tag: $(echo "$PLATFORM_JSON" | jq -r .ANKA_TAG_BASE)::$(echo "$PLATFORM_JSON" | jq -r .HASHED_IMAGE_TAG) always-pull: true debug: true wait-network: true @@ -417,16 +439,10 @@ EOF IFS=$nIFS done IFS=$oIFS - if [[ "$ROUND" != "$ROUNDS" ]]; then - echo ' - wait' - echo '' - fi - done - # Execute multiversion test - if [[ ! "$PINNED" == 'false' || "$SKIP_MULTIVERSION_TEST" == 'false' ]]; then - cat <> ~/.ssh/known_hosts" + - "git clone \$BUILDKITE_REPO ." + - "$GIT_FETCH git checkout -f \$BUILDKITE_COMMIT" + - "echo '+++ :compression: Extracting Test Metrics Code'" + - "tar -zxf .cicd/metrics/test-metrics.tar.gz" + - "echo '+++ :javascript: Running test-metrics.js'" + - "node --max-old-space-size=32768 test-metrics.js" + plugins: + - EOSIO/skip-checkout#v0.1.1: + cd: ~ agents: queue: "$BUILDKITE_TEST_AGENT_QUEUE" timeout: ${TIMEOUT:-10} @@ -524,7 +552,7 @@ cat <(count)/' src/mongocxx/options/change_stream.cpp && \ - sed -i 's/add_subdirectory(test)//' src/mongocxx/CMakeLists.txt src/bsoncxx/CMakeLists.txt && \ - cd build && \ - cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' .. && \ - make -j$(nproc) && \ - make install && \ - cd / && \ - rm -rf mongo-cxx-driver-r3.4.0.tar.gz /mongo-cxx-driver-r3.4.0 -# add mongodb to path -ENV PATH=${PATH}:/mongodb-linux-x86_64-amazon-3.6.3/bin \ No newline at end of file + rm -rf boost_1_72_0.tar.bz2 /boost_1_72_0 +#install libpq postgresql-server +RUN amazon-linux-extras enable postgresql11 && \ + yum install -y libpq-devel postgresql-server && \ + yum clean all && rm -rf /var/cache/yum +#build libpqxx +RUN curl -L https://github.com/jtv/libpqxx/archive/7.2.1.tar.gz | tar zxvf - && \ + cd libpqxx-7.2.1 && \ + cmake -DCMAKE_TOOLCHAIN_FILE=/tmp/clang.cmake -DPostgreSQL_TYPE_INCLUDE_DIR=/usr/include/libpq -DSKIP_BUILD_TEST=ON -DCMAKE_BUILD_TYPE=Release -S . -B build && \ + cmake --build build && cmake --install build && \ + cd .. && rm -rf libpqxx-7.2.1 +ENV PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig +# install nvm +RUN touch ~/.bashrc +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.0/install.sh | bash +# load nvm in non-interactive shells +RUN echo 'export NVM_DIR="$HOME/.nvm"' > ~/.bashrc && \ + echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"' >> ~/.bashrc +# install node 10 +RUN bash -c '. ~/.bashrc; nvm install --lts=dubnium' && \ + ln -s "/root/.nvm/versions/node/$(ls -p /root/.nvm/versions/node | sort -Vr | head -1)bin/node" /usr/local/bin/node && \ + ln -s "/root/.nvm/versions/node/$(ls -p /root/.nvm/versions/node | sort -Vr | head -1)bin/npm" /usr/local/bin/npm +# setup Postgress +RUN su - postgres -c "/usr/bin/initdb" \ No newline at end of file diff --git a/.cicd/platforms/pinned/centos-7.7-pinned.dockerfile b/.cicd/platforms/pinned/centos-7.7-pinned.dockerfile index 00310a96b7a..219a08da5d1 100644 --- a/.cicd/platforms/pinned/centos-7.7-pinned.dockerfile +++ b/.cicd/platforms/pinned/centos-7.7-pinned.dockerfile @@ -6,85 +6,72 @@ RUN yum update -y && \ yum --enablerepo=extras install -y centos-release-scl && \ yum --enablerepo=extras install -y devtoolset-8 && \ yum --enablerepo=extras install -y which git autoconf automake libtool make bzip2 doxygen \ - graphviz bzip2-devel openssl-devel gmp-devel ocaml libicu-devel \ - python python-devel rh-python36 gettext-devel file libusbx-devel \ - libcurl-devel patch vim-common jq -# build cmake. -RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ + graphviz bzip2-devel openssl-devel gmp-devel ocaml \ + python python-devel rh-python36 file libusbx-devel \ + libcurl-devel patch vim-common jq glibc-locale-source glibc-langpack-en && \ + yum clean all && rm -rf /var/cache/yum +# build cmake +RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ + tar -xzf cmake-3.16.2.tar.gz && \ + cd cmake-3.16.2 && \ source /opt/rh/devtoolset-8/enable && \ - source /opt/rh/rh-python36/enable && \ - tar -xzf cmake-3.13.2.tar.gz && \ - cd cmake-3.13.2 && \ ./bootstrap --prefix=/usr/local && \ make -j$(nproc) && \ make install && \ - cd / && \ - rm -rf cmake-3.13.2.tar.gz /cmake-3.13.2 -COPY ./scripts/clang-devtoolset8-support.patch /tmp/clang-devtoolset8-support.patch -# build clang8 -RUN git clone --single-branch --branch release_80 https://github.com/llvm-mirror/llvm.git clang8 && cd clang8 && git checkout 18e41dc && sed -i 's,https://github.com/llvm-mirror/,https://git.llvm.org/git/,g' .git/config && \ - cd tools && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/lld.git && cd lld && git checkout d60a035 && \ - cd ../ && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/polly.git && cd polly && git checkout 1bc06e5 && \ - cd ../ && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/clang.git clang && cd clang && git checkout a03da8b && sed -i 's,https://github.com/llvm-mirror/,https://git.llvm.org/git/,g' .git/config && \ - patch -p2 < /tmp/clang-devtoolset8-support.patch && \ - cd tools && mkdir extra && cd extra && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/clang-tools-extra.git && cd clang-tools-extra && git checkout 6b34834 && \ - cd /clang8/projects && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/libcxx.git && cd libcxx && git checkout 1853712 && \ - cd ../ && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/libcxxabi.git && cd libcxxabi && git checkout d7338a4 && \ - cd ../ && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/libunwind.git && cd libunwind && git checkout 57f6739 && \ - cd ../ && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/compiler-rt.git && cd compiler-rt && git checkout 5bc7979 && \ - mkdir /clang8/build && cd /clang8/build && \ + rm -rf cmake-3.16.2.tar.gz cmake-3.16.2 +# build clang10 +RUN git clone --single-branch --branch llvmorg-10.0.0 https://github.com/llvm/llvm-project clang10 && \ + mkdir /clang10/build && cd /clang10/build && \ source /opt/rh/devtoolset-8/enable && \ source /opt/rh/rh-python36/enable && \ - cmake -G 'Unix Makefiles' -DCMAKE_INSTALL_PREFIX='/usr/local' -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON -DLLVM_BUILD_LLVM_DYLIB=ON -DLLVM_ENABLE_LIBCXX=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_INCLUDE_DOCS=OFF -DLLVM_OPTIMIZED_TABLEGEN=ON -DLLVM_TARGETS_TO_BUILD=X86 -DCMAKE_BUILD_TYPE=Release .. && \ + cmake -G 'Unix Makefiles' -DCMAKE_INSTALL_PREFIX='/usr/local' -DLLVM_ENABLE_PROJECTS='lld;polly;clang;clang-tools-extra;libcxx;libcxxabi;libunwind;compiler-rt' -DLLVM_BUILD_LLVM_DYLIB=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_INCLUDE_DOCS=OFF -DLLVM_TARGETS_TO_BUILD=host -DCMAKE_BUILD_TYPE=Release ../llvm && \ make -j $(nproc) && \ make install && \ cd / && \ - rm -rf /clang8 + rm -rf /clang10 COPY ./.cicd/helpers/clang.make /tmp/clang.cmake -# build llvm8 -RUN git clone --depth 1 --single-branch --branch release_80 https://github.com/llvm-mirror/llvm.git llvm && \ - cd llvm && \ +# build llvm10 +RUN git clone --depth 1 --single-branch --branch llvmorg-10.0.0 https://github.com/llvm/llvm-project llvm && \ + cd llvm/llvm && \ mkdir build && \ cd build && \ - cmake -G 'Unix Makefiles' -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=false -DLLVM_ENABLE_RTTI=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' -DCMAKE_EXE_LINKER_FLAGS=-pthread -DCMAKE_SHARED_LINKER_FLAGS=-pthread -DLLVM_ENABLE_PIC=NO .. && \ + cmake -G 'Unix Makefiles' -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=false -DLLVM_ENABLE_RTTI=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' -DCMAKE_EXE_LINKER_FLAGS=-pthread -DCMAKE_SHARED_LINKER_FLAGS=-pthread -DLLVM_ENABLE_PIC=NO -DLLVM_ENABLE_TERMINFO=OFF .. && \ make -j$(nproc) && \ make install && \ cd / && \ rm -rf /llvm # build boost -RUN curl -LO https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.bz2 && \ - tar -xjf boost_1_71_0.tar.bz2 && \ - cd boost_1_71_0 && \ +RUN curl -LO https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.bz2 && \ + tar -xjf boost_1_72_0.tar.bz2 && \ + cd boost_1_72_0 && \ ./bootstrap.sh --with-toolset=clang --prefix=/usr/local && \ ./b2 toolset=clang cxxflags='-stdlib=libc++ -D__STRICT_ANSI__ -nostdinc++ -I/usr/local/include/c++/v1 -D_FORTIFY_SOURCE=2 -fstack-protector-strong -fpie' linkflags='-stdlib=libc++ -pie' link=static threading=multi --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install && \ cd / && \ - rm -rf boost_1_71_0.tar.bz2 /boost_1_71_0 -# build mongodb -RUN curl -LO https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-amazon-3.6.3.tgz && \ - tar -xzf mongodb-linux-x86_64-amazon-3.6.3.tgz && \ - rm -rf mongodb-linux-x86_64-amazon-3.6.3.tgz -# build mongodb c driver -RUN curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz && \ - tar -xzf mongo-c-driver-1.13.0.tar.gz && \ - cd mongo-c-driver-1.13.0 && \ - mkdir -p build && \ - cd build && \ - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_BSON=ON -DENABLE_SSL=OPENSSL -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON -DENABLE_ICU=OFF -DENABLE_SNAPPY=OFF -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' .. && \ - make -j$(nproc) && \ - make install && \ - cd / && \ - rm -rf mongo-c-driver-1.13.0.tar.gz /mongo-c-driver-1.13.0 -# build mongodb cxx driver -RUN curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz && \ - tar -xzf mongo-cxx-driver-r3.4.0.tar.gz && \ - cd mongo-cxx-driver-r3.4.0 && \ - sed -i 's/\"maxAwaitTimeMS\", count/\"maxAwaitTimeMS\", static_cast(count)/' src/mongocxx/options/change_stream.cpp && \ - sed -i 's/add_subdirectory(test)//' src/mongocxx/CMakeLists.txt src/bsoncxx/CMakeLists.txt && \ - cd build && \ - cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' .. && \ - make -j$(nproc) && \ - make install && \ - cd / && \ - rm -rf mongo-cxx-driver-r3.4.0.tar.gz /mongo-cxx-driver-r3.4.0 -# add mongodb to path -ENV PATH=${PATH}:/mongodb-linux-x86_64-amazon-3.6.3/bin \ No newline at end of file + rm -rf boost_1_72_0.tar.bz2 /boost_1_72_0 +#install libpq +RUN yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm && \ + yum install -y postgresql13-devel postgresql13-server && \ + yum clean all && rm -rf /var/cache/yum +ENV PostgreSQL_ROOT=/usr/pgsql-13 +ENV PKG_CONFIG_PATH=/usr/pgsql-13/lib/pkgconfig:/usr/local/lib64/pkgconfig +#build libpqxx +RUN curl -L https://github.com/jtv/libpqxx/archive/7.2.1.tar.gz | tar zxvf - && \ + cd libpqxx-7.2.1 && \ + cmake -DCMAKE_TOOLCHAIN_FILE=/tmp/clang.cmake -DSKIP_BUILD_TEST=ON -DCMAKE_BUILD_TYPE=Release -S . -B build && \ + cmake --build build && cmake --install build && \ + cd .. && rm -rf libpqxx-7.2.1 +# install nvm +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.0/install.sh | bash +# load nvm in non-interactive shells +RUN cp ~/.bashrc ~/.bashrc.bak && \ + cat ~/.bashrc.bak | tail -3 > ~/.bashrc && \ + cat ~/.bashrc.bak | head -n '-3' >> ~/.bashrc && \ + rm ~/.bashrc.bak +# install node 10 +RUN bash -c '. ~/.bashrc; nvm install --lts=dubnium' && \ + ln -s "/root/.nvm/versions/node/$(ls -p /root/.nvm/versions/node | sort -Vr | head -1)bin/node" /usr/local/bin/node +RUN yum install -y nodejs && \ + yum clean all && rm -rf /var/cache/yum +# setup Postgress +RUN localedef -c -f UTF-8 -i en_US en_US.UTF-8 && \ + su - postgres -c "/usr/pgsql-13/bin/initdb" diff --git a/.cicd/platforms/pinned/centos-8-pinned.dockerfile b/.cicd/platforms/pinned/centos-8-pinned.dockerfile new file mode 100644 index 00000000000..4e4233c921c --- /dev/null +++ b/.cicd/platforms/pinned/centos-8-pinned.dockerfile @@ -0,0 +1,75 @@ +FROM centos:8 +ENV VERSION 1 +#install dependencies +RUN yum update -y && \ + yum install -y epel-release && \ + yum --enablerepo=extras install -y which git autoconf automake libtool make bzip2 && \ + yum --enablerepo=extras install -y graphviz bzip2-devel openssl-devel gmp-devel && \ + yum --enablerepo=extras install -y file libusbx-devel && \ + yum --enablerepo=extras install -y libcurl-devel patch vim-common jq && \ + yum install -y python3 glibc-locale-source glibc-langpack-en && \ + yum clean all && rm -rf /var/cache/yum +RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \ + dnf group install -y "Development Tools" && \ + dnf --enablerepo=powertools install -y doxygen ocaml +# cmake3.18.0 +RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.18.0/cmake-3.18.0.tar.gz && \ + tar -xzf cmake-3.18.0.tar.gz && \ + cd cmake-3.18.0 && \ + ./bootstrap --prefix=/usr/local && \ + make -j$(nproc) && make install && \ + rm -rf cmake-3.18.0.tar.gz cmake-3.18.2 +# clang10.0.0 +RUN git clone --single-branch --branch llvmorg-10.0.0 https://github.com/llvm/llvm-project clang10 && \ + mkdir /clang10/build && cd /clang10/build && \ + cmake -G 'Unix Makefiles' -DCMAKE_INSTALL_PREFIX='/usr/local' -DLLVM_ENABLE_PROJECTS='lld;polly;clang;clang-tools-extra;libcxx;libcxxabi;libunwind;compiler-rt' -DLLVM_BUILD_LLVM_DYLIB=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_INCLUDE_DOCS=OFF -DLLVM_TARGETS_TO_BUILD=host -DCMAKE_BUILD_TYPE=Release ../llvm && \ + make -j$(nproc) && make install && \ + cd / && \ + rm -rf /clang10 +COPY ./.cicd/helpers/clang.make /tmp/clang.cmake +#build llvm10 +RUN git clone --depth 1 --single-branch --branch llvmorg-10.0.0 https://github.com/llvm/llvm-project llvm && \ + cd llvm/llvm && \ + mkdir build && \ + cd build && \ + cmake -G 'Unix Makefiles' -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=false -DLLVM_ENABLE_RTTI=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' -DCMAKE_EXE_LINKER_FLAGS=-pthread -DCMAKE_SHARED_LINKER_FLAGS=-pthread -DLLVM_ENABLE_PIC=NO -DLLVM_ENABLE_TERMINFO=OFF .. && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf /llvm +# build boost +RUN curl -LO https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.bz2 && \ + tar -xjf boost_1_72_0.tar.bz2 && \ + cd boost_1_72_0 && \ + ./bootstrap.sh --with-toolset=clang --prefix=/usr/local && \ + ./b2 toolset=clang cxxflags='-stdlib=libc++ -D__STRICT_ANSI__ -nostdinc++ -I/usr/local/include/c++/v1 -D_FORTIFY_SOURCE=2 -fstack-protector-strong -fpie' linkflags='-stdlib=libc++ -pie' link=static threading=multi --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install && \ + cd / && \ + rm -rf boost_1_72_0.tar.bz2 /boost_1_72_0 +# install libpq & postgres +RUN dnf install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm && \ + dnf -qy module disable postgresql && \ + dnf install -y postgresql13-devel postgresql13-server \ + && dnf clean all && rm -rf /var/cache/yum +ENV PostgreSQL_ROOT=/usr/pgsql-13 +ENV PKG_CONFIG_PATH=/usr/pgsql-13/lib/pkgconfig:/usr/local/lib64/pkgconfig +#build libpqxx +RUN curl -L https://github.com/jtv/libpqxx/archive/7.2.1.tar.gz | tar zxvf - && \ + cd libpqxx-7.2.1 && \ + cmake -DCMAKE_TOOLCHAIN_FILE=/tmp/clang.cmake -DSKIP_BUILD_TEST=ON -DCMAKE_BUILD_TYPE=Release -S . -B build && \ + cmake --build build && cmake --install build && \ + cd .. && rm -rf libpqxx-7.2.1 +# install nvm +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.0/install.sh | bash +# load nvm in non-interactive shells +RUN cp ~/.bashrc ~/.bashrc.bak && \ + cat ~/.bashrc.bak | tail -3 > ~/.bashrc && \ + cat ~/.bashrc.bak | head -n '-3' >> ~/.bashrc && \ + rm ~/.bashrc.bak +# install node 10 +RUN bash -c '. ~/.bashrc; nvm install --lts=dubnium' && \ + ln -s "/root/.nvm/versions/node/$(ls -p /root/.nvm/versions/node | sort -Vr | head -1)bin/node" /usr/local/bin/node +RUN yum install -y nodejs && \ + yum clean all && rm -rf /var/cache/yum +# setup Postgress +RUN localedef -c -f UTF-8 -i en_US en_US.UTF-8 && \ + su - postgres -c "/usr/pgsql-13/bin/initdb" \ No newline at end of file diff --git a/.cicd/platforms/pinned/macos-10.14-pinned.sh b/.cicd/platforms/pinned/macos-10.14-pinned.sh index 779a566adb0..ad8baf25c73 100755 --- a/.cicd/platforms/pinned/macos-10.14-pinned.sh +++ b/.cicd/platforms/pinned/macos-10.14-pinned.sh @@ -2,89 +2,36 @@ set -eo pipefail VERSION=1 brew update -brew install git cmake python libtool libusb graphviz automake wget gmp llvm@7 pkgconfig doxygen openssl@1.1 jq || : +brew install git cmake python libtool libusb graphviz automake wget gmp pkgconfig doxygen openssl@1.1 jq libpq postgres || : # install clang from source -git clone --single-branch --branch release_80 https://github.com/llvm-mirror/llvm.git clang8 -cd clang8 -git checkout 18e41dc -sed -i.bak 's,https://github.com/llvm-mirror/,https://git.llvm.org/git/,g' .git/config -cd tools -git clone --single-branch --branch release_80 https://github.com/llvm-mirror/lld.git -cd lld -git checkout d60a035 -cd ../ -git clone --single-branch --branch release_80 https://github.com/llvm-mirror/polly.git -cd polly -git checkout 1bc06e5 -cd ../ -git clone --single-branch --branch release_80 https://github.com/llvm-mirror/clang.git clang -cd clang -git checkout a03da8b -sed -i.bak 's,https://github.com/llvm-mirror/,https://git.llvm.org/git/,g' .git/config -cd tools -mkdir extra -cd extra -git clone --single-branch --branch release_80 https://github.com/llvm-mirror/clang-tools-extra.git -cd clang-tools-extra -git checkout 6b34834 -cd ../../../../../projects/ -git clone --single-branch --branch release_80 https://github.com/llvm-mirror/libcxx.git -cd libcxx -git checkout 1853712 -cd ../ -git clone --single-branch --branch release_80 https://github.com/llvm-mirror/libcxxabi.git -cd libcxxabi -git checkout d7338a4 -cd ../ -git clone --single-branch --branch release_80 https://github.com/llvm-mirror/libunwind.git -cd libunwind -git checkout 57f6739 -cd ../ -git clone --single-branch --branch release_80 https://github.com/llvm-mirror/compiler-rt.git -cd compiler-rt -git checkout 5bc7979 -mkdir ../../build -cd ../../build -cmake -G 'Unix Makefiles' -DCMAKE_INSTALL_PREFIX='/usr/local' -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON -DLLVM_BUILD_LLVM_DYLIB=ON -DLLVM_ENABLE_LIBCXX=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_INCLUDE_DOCS=OFF -DLLVM_OPTIMIZED_TABLEGEN=ON -DLLVM_TARGETS_TO_BUILD=X86 -DCMAKE_BUILD_TYPE=Release .. +git clone --single-branch --branch llvmorg-10.0.0 https://github.com/llvm/llvm-project clang10 +mkdir clang10/build +cd clang10/build +cmake -G 'Unix Makefiles' -DCMAKE_INSTALL_PREFIX='/usr/local' -DLLVM_ENABLE_PROJECTS='lld;polly;clang;clang-tools-extra;libcxx;libcxxabi;libunwind;compiler-rt' -DLLVM_BUILD_LLVM_DYLIB=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_INCLUDE_DOCS=OFF -DLLVM_TARGETS_TO_BUILD=host -DCMAKE_BUILD_TYPE=Release ../llvm && \ make -j $(getconf _NPROCESSORS_ONLN) sudo make install cd ../.. -rm -rf clang8 +rm -rf clang10 # install boost from source -# Boost Fix: eosio/install/bin/../include/c++/v1/stdlib.h:94:15: fatal error: 'stdlib.h' file not found +## Boost Fix: eosio/install/bin/../include/c++/v1/stdlib.h:94:15: fatal error: 'stdlib.h' file not found export SDKROOT="$(xcrun --sdk macosx --show-sdk-path)" -curl -LO https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.bz2 -tar -xjf boost_1_71_0.tar.bz2 -cd boost_1_71_0 +curl -LO https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.bz2 +tar -xjf boost_1_72_0.tar.bz2 +cd boost_1_72_0 ./bootstrap.sh --prefix=/usr/local sudo SDKROOT="$SDKROOT" ./b2 --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(getconf _NPROCESSORS_ONLN) install cd .. -sudo rm -rf boost_1_71_0.tar.bz2 boost_1_71_0 -# install mongoDB -cd ~ -curl -OL https://fastdl.mongodb.org/osx/mongodb-osx-ssl-x86_64-3.6.3.tgz -tar -xzf mongodb-osx-ssl-x86_64-3.6.3.tgz -rm -f mongodb-osx-ssl-x86_64-3.6.3.tgz -ln -s ~/mongodb-osx-x86_64-3.6.3 ~/mongodb -# install mongo-c-driver from source -cd /tmp -curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz -tar -xzf mongo-c-driver-1.13.0.tar.gz -cd mongo-c-driver-1.13.0 -mkdir -p cmake-build -cd cmake-build -cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX='/usr/local' -DENABLE_BSON=ON -DENABLE_SSL=DARWIN -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON -DENABLE_ICU=OFF -DENABLE_SASL=OFF -DENABLE_SNAPPY=OFF .. -make -j $(getconf _NPROCESSORS_ONLN) -sudo make install -cd ../.. -rm mongo-c-driver-1.13.0.tar.gz -# install mongo-cxx-driver from source -cd /tmp -curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz -tar -xzf mongo-cxx-driver-r3.4.0.tar.gz -cd mongo-cxx-driver-r3.4.0/build -cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX='/usr/local' .. -make -j $(getconf _NPROCESSORS_ONLN) VERBOSE=1 -sudo make install -cd ../.. -rm -f mongo-cxx-driver-r3.4.0.tar.gz \ No newline at end of file +sudo rm -rf boost_1_72_0.tar.bz2 boost_1_72_0 + +# install libpqxx from source +curl -L https://github.com/jtv/libpqxx/archive/7.2.1.tar.gz | tar zxvf - +cd libpqxx-7.2.1 +cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DPostgreSQL_ROOT=/usr/local/opt/libpq -DSKIP_BUILD_TEST=ON -DCMAKE_BUILD_TYPE=Release -S . -B build +cmake --build build && cmake --install build +cd .. && rm -rf libpqxx-7.2.1 + +# install nvm for ship_test +cd ~ && brew install nvm && mkdir -p ~/.nvm && echo "export NVM_DIR=$HOME/.nvm" >> ~/.bash_profile && echo 'source $(brew --prefix nvm)/nvm.sh' >> ~/.bash_profile && cat ~/.bash_profile && source ~/.bash_profile && echo $NVM_DIR && nvm install --lts=dubnium +# initialize postgres configuration files +sudo rm -rf /usr/local/var/postgres +initdb --locale=C -E UTF-8 /usr/local/var/postgres diff --git a/.cicd/platforms/pinned/macos-10.15-pinned.sh b/.cicd/platforms/pinned/macos-10.15-pinned.sh new file mode 100755 index 00000000000..1f7d043d5f4 --- /dev/null +++ b/.cicd/platforms/pinned/macos-10.15-pinned.sh @@ -0,0 +1,36 @@ +#!/bin/bash +set -eo pipefail +VERSION=1 +export SDKROOT="$(xcrun --sdk macosx --show-sdk-path)" +brew update +brew install git cmake python libtool libusb graphviz automake wget gmp pkgconfig doxygen openssl jq postgres || : +# install clang from source +git clone --single-branch --branch llvmorg-10.0.0 https://github.com/llvm/llvm-project clang10 +mkdir clang10/build +cd clang10/build +cmake -G 'Unix Makefiles' -DCMAKE_INSTALL_PREFIX='/usr/local' -DLLVM_ENABLE_PROJECTS='lld;polly;clang;clang-tools-extra;libcxx;libcxxabi;libunwind;compiler-rt' -DLLVM_BUILD_LLVM_DYLIB=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_INCLUDE_DOCS=OFF -DLLVM_TARGETS_TO_BUILD=host -DCMAKE_BUILD_TYPE=Release ../llvm && \ +make -j $(getconf _NPROCESSORS_ONLN) +sudo make install +cd ../.. +rm -rf clang10 +# install boost from source +curl -LO https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.bz2 +tar -xjf boost_1_72_0.tar.bz2 +cd boost_1_72_0 +./bootstrap.sh --prefix=/usr/local +sudo -E ./b2 --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(getconf _NPROCESSORS_ONLN) install +cd .. +sudo rm -rf boost_1_72_0.tar.bz2 boost_1_72_0 + +# install libpqxx from source +curl -L https://github.com/jtv/libpqxx/archive/7.2.1.tar.gz | tar zxvf - +cd libpqxx-7.2.1 +cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DPostgreSQL_ROOT=/usr/local/opt/libpq -DSKIP_BUILD_TEST=ON -DCMAKE_BUILD_TYPE=Release -S . -B build +cmake --build build && cmake --install build +cd .. && rm -rf libpqxx-7.2.1 + +# install nvm for ship_test +cd ~ && brew install nvm && mkdir -p ~/.nvm && echo "export NVM_DIR=$HOME/.nvm" >> ~/.bash_profile && echo 'source $(brew --prefix nvm)/nvm.sh' >> ~/.bash_profile && cat ~/.bash_profile && source ~/.bash_profile && echo $NVM_DIR && nvm install --lts=dubnium +# initialize postgres configuration files +sudo rm -rf /usr/local/var/postgres +initdb --locale=C -E UTF-8 /usr/local/var/postgres diff --git a/.cicd/platforms/pinned/ubuntu-16.04-pinned.dockerfile b/.cicd/platforms/pinned/ubuntu-16.04-pinned.dockerfile index 012e5ea4688..45d06c8bb11 100644 --- a/.cicd/platforms/pinned/ubuntu-16.04-pinned.dockerfile +++ b/.cicd/platforms/pinned/ubuntu-16.04-pinned.dockerfile @@ -4,79 +4,69 @@ ENV VERSION 1 RUN apt-get update && \ apt-get upgrade -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential git automake \ - libbz2-dev libssl-dev doxygen graphviz libgmp3-dev autotools-dev libicu-dev \ + libbz2-dev libssl-dev doxygen graphviz libgmp3-dev autotools-dev \ python2.7 python2.7-dev python3 python3-dev autoconf libtool curl zlib1g-dev \ sudo ruby libusb-1.0-0-dev libcurl4-gnutls-dev pkg-config apt-transport-https vim-common jq -# build cmake. -RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ - tar -xzf cmake-3.13.2.tar.gz && \ - cd cmake-3.13.2 && \ +# build cmake +RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ + tar -xzf cmake-3.16.2.tar.gz && \ + cd cmake-3.16.2 && \ ./bootstrap --prefix=/usr/local && \ make -j$(nproc) && \ make install && \ - cd / && \ - rm -rf cmake-3.13.2.tar.gz /cmake-3.13.2 -# build clang -RUN git clone --single-branch --branch release_80 https://github.com/llvm-mirror/llvm.git clang8 && cd clang8 && git checkout 18e41dc && sed -i 's,https://github.com/llvm-mirror/,https://git.llvm.org/git/,g' .git/config && \ - cd tools && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/lld.git && cd lld && git checkout d60a035 && \ - cd ../ && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/polly.git && cd polly && git checkout 1bc06e5 && \ - cd ../ && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/clang.git clang && cd clang && git checkout a03da8b && sed -i 's,https://github.com/llvm-mirror/,https://git.llvm.org/git/,g' .git/config && \ - cd tools && mkdir extra && cd extra && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/clang-tools-extra.git && cd clang-tools-extra && git checkout 6b34834 && \ - cd /clang8/projects && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/libcxx.git && cd libcxx && git checkout 1853712 && \ - cd ../ && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/libcxxabi.git && cd libcxxabi && git checkout d7338a4 && \ - cd ../ && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/libunwind.git && cd libunwind && git checkout 57f6739 && \ - cd ../ && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/compiler-rt.git && cd compiler-rt && git checkout 5bc7979 && \ - mkdir /clang8/build && cd /clang8/build && \ - cmake -G 'Unix Makefiles' -DCMAKE_INSTALL_PREFIX='/usr/local' -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON -DLLVM_BUILD_LLVM_DYLIB=ON -DLLVM_ENABLE_LIBCXX=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_INCLUDE_DOCS=OFF -DLLVM_OPTIMIZED_TABLEGEN=ON -DLLVM_TARGETS_TO_BUILD=X86 -DCMAKE_BUILD_TYPE=Release .. && \ + rm -rf cmake-3.16.2.tar.gz cmake-3.16.2 +# build clang10 +RUN git clone --single-branch --branch llvmorg-10.0.0 https://github.com/llvm/llvm-project clang10 && \ + mkdir /clang10/build && cd /clang10/build && \ + cmake -G 'Unix Makefiles' -DCMAKE_INSTALL_PREFIX='/usr/local' -DLLVM_ENABLE_PROJECTS='lld;polly;clang;clang-tools-extra;libcxx;libcxxabi;libunwind;compiler-rt' -DLLVM_BUILD_LLVM_DYLIB=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_INCLUDE_DOCS=OFF -DLLVM_TARGETS_TO_BUILD=host -DCMAKE_BUILD_TYPE=Release ../llvm && \ make -j $(nproc) && \ make install && \ cd / && \ - rm -rf /clang8 + rm -rf /clang10 COPY ./.cicd/helpers/clang.make /tmp/clang.cmake -# build llvm8 -RUN git clone --depth 1 --single-branch --branch release_80 https://github.com/llvm-mirror/llvm.git llvm && \ - cd llvm && \ +# build llvm10 +RUN git clone --depth 1 --single-branch --branch llvmorg-10.0.0 https://github.com/llvm/llvm-project llvm && \ + cd llvm/llvm && \ mkdir build && \ cd build && \ - cmake -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=false -DLLVM_ENABLE_RTTI=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' -DCMAKE_EXE_LINKER_FLAGS=-pthread -DCMAKE_SHARED_LINKER_FLAGS=-pthread -DLLVM_ENABLE_PIC=NO .. && \ + cmake -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=false -DLLVM_ENABLE_RTTI=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' -DCMAKE_EXE_LINKER_FLAGS=-pthread -DCMAKE_SHARED_LINKER_FLAGS=-pthread -DLLVM_ENABLE_PIC=NO -DLLVM_ENABLE_TERMINFO=OFF .. && \ make -j$(nproc) && \ make install && \ cd / && \ rm -rf /llvm # build boost -RUN curl -LO https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.bz2 && \ - tar -xjf boost_1_71_0.tar.bz2 && \ - cd boost_1_71_0 && \ +RUN curl -LO https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.bz2 && \ + tar -xjf boost_1_72_0.tar.bz2 && \ + cd boost_1_72_0 && \ ./bootstrap.sh --with-toolset=clang --prefix=/usr/local && \ ./b2 toolset=clang cxxflags='-stdlib=libc++ -D__STRICT_ANSI__ -nostdinc++ -I/usr/local/include/c++/v1 -D_FORTIFY_SOURCE=2 -fstack-protector-strong -fpie' linkflags='-stdlib=libc++ -pie' link=static threading=multi --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install && \ cd / && \ - rm -rf boost_1_71_0.tar.bz2 /boost_1_71_0 -# build mongodb -RUN curl -LO http://downloads.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1604-3.6.3.tgz && \ - tar -xzf mongodb-linux-x86_64-ubuntu1604-3.6.3.tgz && \ - rm -f mongodb-linux-x86_64-ubuntu1604-3.6.3.tgz -# build mongodb c driver -RUN curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz && \ - tar -xzf mongo-c-driver-1.13.0.tar.gz && \ - cd mongo-c-driver-1.13.0 && \ - mkdir -p build && \ - cd build && \ - cmake --DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_BSON=ON -DENABLE_SSL=OPENSSL -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' .. && \ - make -j$(nproc) && \ - make install && \ - cd / && \ - rm -rf mongo-c-driver-1.13.0.tar.gz /mongo-c-driver-1.13.0 -# build mongodb cxx driver -RUN curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz && \ - tar -xzf mongo-cxx-driver-r3.4.0.tar.gz && \ - cd mongo-cxx-driver-r3.4.0 && \ - sed -i 's/\"maxAwaitTimeMS\", count/\"maxAwaitTimeMS\", static_cast(count)/' src/mongocxx/options/change_stream.cpp && \ - sed -i 's/add_subdirectory(test)//' src/mongocxx/CMakeLists.txt src/bsoncxx/CMakeLists.txt && \ - cd build && \ - cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' .. && \ - make -j$(nproc) && \ - make install && \ - cd / && \ - rm -rf mongo-cxx-driver-r3.4.0.tar.gz /mongo-cxx-driver-r3.4.0 -# add mongodb to path -ENV PATH=${PATH}:/mongodb-linux-x86_64-ubuntu1604-3.6.3/bin \ No newline at end of file + rm -rf boost_1_72_0.tar.bz2 /boost_1_72_0 +# install libpq, postgresql-13 +ENV TZ=America/Chicago +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \ + echo "deb http://apt.postgresql.org/pub/repos/apt xenial-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \ + curl -sL https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \ + apt-get update && apt-get -y install libpq-dev postgresql-13 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* +# build libpqxx +RUN curl -L https://github.com/jtv/libpqxx/archive/7.2.1.tar.gz | tar zxvf - && \ + cd libpqxx-7.2.1 && \ + cmake -DCMAKE_TOOLCHAIN_FILE=/tmp/clang.cmake -DSKIP_BUILD_TEST=ON -DPostgreSQL_TYPE_INCLUDE_DIR=/usr/include/postgresql -DCMAKE_BUILD_TYPE=Release -S . -B build && \ + cmake --build build && cmake --install build && \ + cd .. && rm -rf libpqxx-7.2.1 +#install nvm +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.0/install.sh | bash +# load nvm in non-interactive shells +RUN cp ~/.bashrc ~/.bashrc.bak && \ + cat ~/.bashrc.bak | tail -3 > ~/.bashrc && \ + cat ~/.bashrc.bak | head -n '-3' >> ~/.bashrc && \ + rm ~/.bashrc.bak +# install node 10 +RUN bash -c '. ~/.bashrc; nvm install --lts=dubnium' && \ + ln -s "/root/.nvm/versions/node/$(ls -p /root/.nvm/versions/node | sort -Vr | head -1)bin/node" /usr/local/bin/node +RUN curl -sL https://deb.nodesource.com/setup_13.x | sudo -E bash - +RUN apt-get update && apt-get install -y nodejs && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* diff --git a/.cicd/platforms/pinned/ubuntu-18.04-pinned.dockerfile b/.cicd/platforms/pinned/ubuntu-18.04-pinned.dockerfile index b76efc04dd4..b41f3e2176a 100644 --- a/.cicd/platforms/pinned/ubuntu-18.04-pinned.dockerfile +++ b/.cicd/platforms/pinned/ubuntu-18.04-pinned.dockerfile @@ -5,80 +5,74 @@ RUN apt-get update && \ apt-get upgrade -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y git make \ bzip2 automake libbz2-dev libssl-dev doxygen graphviz libgmp3-dev \ - autotools-dev libicu-dev python2.7 python2.7-dev python3 \ + autotools-dev python2.7 python2.7-dev python3 \ python3-dev python-configparser python-requests python-pip \ - autoconf libtool g++ gcc curl zlib1g-dev sudo ruby libusb-1.0-0-dev \ - libcurl4-gnutls-dev pkg-config patch vim-common jq -# build cmake. -RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ - tar -xzf cmake-3.13.2.tar.gz && \ - cd cmake-3.13.2 && \ + autoconf libtool g++ gcc curl zlib1g-dev sudo ruby libusb-1.0-0-dev\ + libcurl4-gnutls-dev pkg-config patch vim-common jq && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* +# build cmake +RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ + tar -xzf cmake-3.16.2.tar.gz && \ + cd cmake-3.16.2 && \ ./bootstrap --prefix=/usr/local && \ make -j$(nproc) && \ make install && \ - cd / && \ - rm -rf cmake-3.13.2.tar.gz /cmake-3.13.2 -# build clang8 -RUN git clone --single-branch --branch release_80 https://github.com/llvm-mirror/llvm.git clang8 && cd clang8 && git checkout 18e41dc && sed -i 's,https://github.com/llvm-mirror/,https://git.llvm.org/git/,g' .git/config && \ - cd tools && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/lld.git && cd lld && git checkout d60a035 && \ - cd ../ && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/polly.git && cd polly && git checkout 1bc06e5 && \ - cd ../ && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/clang.git clang && cd clang && git checkout a03da8b && sed -i 's,https://github.com/llvm-mirror/,https://git.llvm.org/git/,g' .git/config && \ - cd tools && mkdir extra && cd extra && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/clang-tools-extra.git && cd clang-tools-extra && git checkout 6b34834 && \ - cd /clang8/projects && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/libcxx.git && cd libcxx && git checkout 1853712 && \ - cd ../ && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/libcxxabi.git && cd libcxxabi && git checkout d7338a4 && \ - cd ../ && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/libunwind.git && cd libunwind && git checkout 57f6739 && \ - cd ../ && git clone --single-branch --branch release_80 https://github.com/llvm-mirror/compiler-rt.git && cd compiler-rt && git checkout 5bc7979 && \ - mkdir /clang8/build && cd /clang8/build && \ - cmake -G 'Unix Makefiles' -DCMAKE_INSTALL_PREFIX='/usr/local' -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON -DLLVM_BUILD_LLVM_DYLIB=ON -DLLVM_ENABLE_LIBCXX=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_INCLUDE_DOCS=OFF -DLLVM_OPTIMIZED_TABLEGEN=ON -DLLVM_TARGETS_TO_BUILD=X86 -DCMAKE_BUILD_TYPE=Release .. && \ + rm -rf cmake-3.16.2.tar.gz cmake-3.16.2 + +# build clang10 +RUN git clone --single-branch --branch llvmorg-10.0.0 https://github.com/llvm/llvm-project clang10 && \ + mkdir /clang10/build && cd /clang10/build && \ + cmake -G 'Unix Makefiles' -DCMAKE_INSTALL_PREFIX='/usr/local' -DLLVM_ENABLE_PROJECTS='lld;polly;clang;clang-tools-extra;libcxx;libcxxabi;libunwind;compiler-rt' -DLLVM_BUILD_LLVM_DYLIB=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_INCLUDE_DOCS=OFF -DLLVM_TARGETS_TO_BUILD=host -DCMAKE_BUILD_TYPE=Release ../llvm && \ make -j $(nproc) && \ make install && \ cd / && \ - rm -rf /clang8 + rm -rf /clang10 COPY ./.cicd/helpers/clang.make /tmp/clang.cmake -# build llvm8 -RUN git clone --depth 1 --single-branch --branch release_80 https://github.com/llvm-mirror/llvm.git llvm && \ - cd llvm && \ +# build llvm10 +RUN git clone --depth 1 --single-branch --branch llvmorg-10.0.0 https://github.com/llvm/llvm-project llvm && \ + cd llvm/llvm && \ mkdir build && \ cd build && \ - cmake -G 'Unix Makefiles' -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=false -DLLVM_ENABLE_RTTI=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' -DCMAKE_EXE_LINKER_FLAGS=-pthread -DCMAKE_SHARED_LINKER_FLAGS=-pthread -DLLVM_ENABLE_PIC=NO .. && \ + cmake -G 'Unix Makefiles' -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=false -DLLVM_ENABLE_RTTI=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' -DCMAKE_EXE_LINKER_FLAGS=-pthread -DCMAKE_SHARED_LINKER_FLAGS=-pthread -DLLVM_ENABLE_PIC=NO -DLLVM_ENABLE_TERMINFO=OFF .. && \ make -j$(nproc) && \ make install && \ cd / && \ rm -rf /llvm # build boost -RUN curl -LO https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.bz2 && \ - tar -xjf boost_1_71_0.tar.bz2 && \ - cd boost_1_71_0 && \ +RUN curl -LO https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.bz2 && \ + tar -xjf boost_1_72_0.tar.bz2 && \ + cd boost_1_72_0 && \ ./bootstrap.sh --with-toolset=clang --prefix=/usr/local && \ ./b2 toolset=clang cxxflags='-stdlib=libc++ -D__STRICT_ANSI__ -nostdinc++ -I/usr/local/include/c++/v1 -D_FORTIFY_SOURCE=2 -fstack-protector-strong -fpie' linkflags='-stdlib=libc++ -pie' link=static threading=multi --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install && \ cd / && \ - rm -rf boost_1_71_0.tar.bz2 /boost_1_71_0 -# build mongodb -RUN curl -LO http://downloads.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1804-4.1.1.tgz && \ - tar -xzf mongodb-linux-x86_64-ubuntu1804-4.1.1.tgz && \ - rm -f mongodb-linux-x86_64-ubuntu1804-4.1.1.tgz -# build mongodb c driver -RUN curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz && \ - tar -xzf mongo-c-driver-1.13.0.tar.gz && \ - cd mongo-c-driver-1.13.0 && \ - mkdir -p build && \ - cd build && \ - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_BSON=ON -DENABLE_SSL=OPENSSL -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' .. && \ - make -j$(nproc) && \ - make install && \ - cd / && \ - rm -rf mongo-c-driver-1.13.0.tar.gz /mongo-c-driver-1.13.0 -# build mongodb cxx driver -RUN curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz && \ - tar -xzf mongo-cxx-driver-r3.4.0.tar.gz && \ - cd mongo-cxx-driver-r3.4.0 && \ - sed -i 's/\"maxAwaitTimeMS\", count/\"maxAwaitTimeMS\", static_cast(count)/' src/mongocxx/options/change_stream.cpp && \ - sed -i 's/add_subdirectory(test)//' src/mongocxx/CMakeLists.txt src/bsoncxx/CMakeLists.txt && \ - cd build && \ - cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' .. && \ - make -j$(nproc) && \ - make install && \ - cd / && \ - rm -rf mongo-cxx-driver-r3.4.0.tar.gz /mongo-cxx-driver-r3.4.0 -# add mongodb to path -ENV PATH=${PATH}:/mongodb-linux-x86_64-ubuntu1804-4.1.1/bin \ No newline at end of file + rm -rf boost_1_72_0.tar.bz2 /boost_1_72_0 +# install libpq, postgresql-13 +ENV TZ=America/Chicago +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \ + echo "deb http://apt.postgresql.org/pub/repos/apt bionic-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \ + curl -sL https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \ + apt-get update && apt-get -y install libpq-dev postgresql-13 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* +#build libpqxx +RUN curl -L https://github.com/jtv/libpqxx/archive/7.2.1.tar.gz | tar zxvf - && \ + cd libpqxx-7.2.1 && \ + cmake -DCMAKE_TOOLCHAIN_FILE=/tmp/clang.cmake -DSKIP_BUILD_TEST=ON -DPostgreSQL_TYPE_INCLUDE_DIR=/usr/include/postgresql -DCMAKE_BUILD_TYPE=Release -S . -B build && \ + cmake --build build && cmake --install build && \ + cd .. && rm -rf libpqxx-7.2.1 + +# install nvm +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.0/install.sh | bash +# load nvm in non-interactive shells +RUN cp ~/.bashrc ~/.bashrc.bak && \ + cat ~/.bashrc.bak | tail -3 > ~/.bashrc && \ + cat ~/.bashrc.bak | head -n '-3' >> ~/.bashrc && \ + rm ~/.bashrc.bak +# install node 10 +RUN bash -c '. ~/.bashrc; nvm install --lts=dubnium' && \ + ln -s "/root/.nvm/versions/node/$(ls -p /root/.nvm/versions/node | sort -Vr | head -1)bin/node" /usr/local/bin/node +RUN curl -sL https://deb.nodesource.com/setup_13.x | sudo -E bash - +RUN apt-get update && apt-get install -y nodejs && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* \ No newline at end of file diff --git a/.cicd/platforms/pinned/ubuntu-20.04-pinned.dockerfile b/.cicd/platforms/pinned/ubuntu-20.04-pinned.dockerfile new file mode 100644 index 00000000000..9762e0e6fe4 --- /dev/null +++ b/.cicd/platforms/pinned/ubuntu-20.04-pinned.dockerfile @@ -0,0 +1,72 @@ +FROM ubuntu:20.04 +ENV VERSION 1 +# install dependencies. +RUN apt-get update && \ + apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y git make \ + bzip2 automake libbz2-dev libssl-dev doxygen graphviz libgmp3-dev \ + autotools-dev python2.7 python2.7-dev python3 \ + python3-dev python-configparser \ + autoconf libtool g++ gcc curl zlib1g-dev sudo ruby libusb-1.0-0-dev \ + libcurl4-gnutls-dev pkg-config patch vim-common jq gnupg && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# build cmake +RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ + tar -xzf cmake-3.16.2.tar.gz && \ + cd cmake-3.16.2 && \ + ./bootstrap --prefix=/usr/local && \ + make -j$(nproc) && \ + make install && \ + rm -rf cmake-3.16.2.tar.gz cmake-3.16.2 +# build clang10 +RUN git clone --single-branch --branch llvmorg-10.0.0 https://github.com/llvm/llvm-project clang10 && \ + mkdir /clang10/build && cd /clang10/build && \ + cmake -G 'Unix Makefiles' -DCMAKE_INSTALL_PREFIX='/usr/local' -DLLVM_ENABLE_PROJECTS='lld;polly;clang;clang-tools-extra;libcxx;libcxxabi;libunwind;compiler-rt' -DLLVM_BUILD_LLVM_DYLIB=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_INCLUDE_DOCS=OFF -DLLVM_TARGETS_TO_BUILD=host -DCMAKE_BUILD_TYPE=Release ../llvm && \ + make -j $(nproc) && \ + make install && \ + cd / && \ + rm -rf /clang10 +COPY ./.cicd/helpers/clang.make /tmp/clang.cmake +# build llvm10 +RUN git clone --depth 1 --single-branch --branch llvmorg-10.0.0 https://github.com/llvm/llvm-project llvm && \ + cd llvm/llvm && \ + mkdir build && \ + cd build && \ + cmake -G 'Unix Makefiles' -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=false -DLLVM_ENABLE_RTTI=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' -DCMAKE_EXE_LINKER_FLAGS=-pthread -DCMAKE_SHARED_LINKER_FLAGS=-pthread -DLLVM_ENABLE_PIC=NO -DLLVM_ENABLE_TERMINFO=OFF .. && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf /llvm +# install libpq postgresql +ENV TZ=America/Chicago +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \ + echo "deb http://apt.postgresql.org/pub/repos/apt focal-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \ + curl -sL https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \ + apt-get update && apt-get -y install libpq-dev postgresql-13 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* +#build libpqxx +RUN curl -L https://github.com/jtv/libpqxx/archive/7.2.1.tar.gz | tar zxvf - && \ + cd libpqxx-7.2.1 && \ + cmake -DCMAKE_TOOLCHAIN_FILE=/tmp/clang.cmake -DSKIP_BUILD_TEST=ON -DPostgreSQL_TYPE_INCLUDE_DIR=/usr/include/postgresql -DCMAKE_BUILD_TYPE=Release -S . -B build && \ + cmake --build build && cmake --install build && \ + cd .. && rm -rf libpqxx-7.2.1 +# build boost +RUN curl -LO https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.bz2 && \ + tar -xjf boost_1_72_0.tar.bz2 && \ + cd boost_1_72_0 && \ + ./bootstrap.sh --with-toolset=clang --prefix=/usr/local && \ + ./b2 toolset=clang cxxflags='-stdlib=libc++ -D__STRICT_ANSI__ -nostdinc++ -I/usr/local/include/c++/v1 -D_FORTIFY_SOURCE=2 -fstack-protector-strong -fpie' linkflags='-stdlib=libc++ -pie' link=static threading=multi --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install && \ + cd / && \ + rm -rf boost_1_72_0.tar.bz2 /boost_1_72_0 +# install node 12 +RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \ + . /etc/lsb-release && \ + echo "deb https://deb.nodesource.com/node_12.x $DISTRIB_CODENAME main" | tee /etc/apt/sources.list.d/nodesource.list && \ + echo "deb-src https://deb.nodesource.com/node_12.x $DISTRIB_CODENAME main" | tee -a /etc/apt/sources.list.d/nodesource.list && \ + apt-get update && \ + apt-get install -y nodejs && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* diff --git a/.cicd/platforms/unpinned/amazon_linux-2-unpinned.dockerfile b/.cicd/platforms/unpinned/amazon_linux-2-unpinned.dockerfile index 38af1dbb519..00a9004932e 100644 --- a/.cicd/platforms/unpinned/amazon_linux-2-unpinned.dockerfile +++ b/.cicd/platforms/unpinned/amazon_linux-2-unpinned.dockerfile @@ -5,16 +5,15 @@ RUN yum update -y && \ yum install -y which git sudo procps-ng util-linux autoconf automake \ libtool make bzip2 bzip2-devel openssl-devel gmp-devel libstdc++ libcurl-devel \ libusbx-devel python3 python3-devel python-devel libedit-devel doxygen \ - graphviz clang patch llvm-devel llvm-static vim-common jq -# build cmake. -RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ - tar -xzf cmake-3.13.2.tar.gz && \ - cd cmake-3.13.2 && \ + graphviz clang patch llvm-devel llvm-static vim-common jq && \ + yum clean all && rm -rf /var/cache/yum +RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ + tar -xzf cmake-3.16.2.tar.gz && \ + cd cmake-3.16.2 && \ ./bootstrap --prefix=/usr/local && \ make -j$(nproc) && \ make install && \ - cd / && \ - rm -rf cmake-3.13.2.tar.gz /cmake-3.13.2 + rm -rf cmake-3.16.2.tar.gz cmake-3.16.2 # build boost RUN curl -LO https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.bz2 && \ tar -xjf boost_1_71_0.tar.bz2 && \ @@ -23,32 +22,25 @@ RUN curl -LO https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/bo ./b2 --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install && \ cd / && \ rm -rf boost_1_71_0.tar.bz2 /boost_1_71_0 -# build mongodb -RUN curl -LO https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-amazon-3.6.3.tgz && \ - tar -xzf mongodb-linux-x86_64-amazon-3.6.3.tgz && \ - rm -f mongodb-linux-x86_64-amazon-3.6.3.tgz -# build mongodb c driver -RUN curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz && \ - tar -xzf mongo-c-driver-1.13.0.tar.gz && \ - cd mongo-c-driver-1.13.0 && \ - mkdir -p build && \ - cd build && \ - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_BSON=ON -DENABLE_SSL=OPENSSL -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON -DENABLE_ICU=OFF -DENABLE_SNAPPY=OFF .. && \ - make -j$(nproc) && \ - make install && \ - cd / && \ - rm -rf mongo-c-driver-1.13.0.tar.gz /mongo-c-driver-1.13.0 -# build mongodb cxx driver -RUN curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz && \ - tar -xzf mongo-cxx-driver-r3.4.0.tar.gz && \ - cd mongo-cxx-driver-r3.4.0 && \ - sed -i 's/\"maxAwaitTimeMS\", count/\"maxAwaitTimeMS\", static_cast(count)/' src/mongocxx/options/change_stream.cpp && \ - sed -i 's/add_subdirectory(test)//' src/mongocxx/CMakeLists.txt src/bsoncxx/CMakeLists.txt && \ - cd build && \ - cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. && \ - make -j$(nproc) && \ - make install && \ - cd / && \ - rm -rf mongo-cxx-driver-r3.4.0.tar.gz /mongo-cxx-driver-r3.4.0 -# add mongodb to path -ENV PATH=${PATH}:/mongodb-linux-x86_64-amazon-3.6.3/bin \ No newline at end of file +#install libpq +RUN amazon-linux-extras enable postgresql11 && \ + yum install -y libpq-devel postgresql-server && \ + yum clean all && rm -rf /var/cache/yum +#build libpqxx +RUN curl -L https://github.com/jtv/libpqxx/archive/7.2.1.tar.gz | tar zxvf - && \ + cd libpqxx-7.2.1 && \ + cmake -DSKIP_BUILD_TEST=ON -DPostgreSQL_TYPE_INCLUDE_DIR=/usr/include/libpq -DCMAKE_BUILD_TYPE=Release -S . -B build && \ + cmake --build build && cmake --install build && \ + cd .. && rm -rf libpqxx-7.2.1 +ENV PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig +# install nvm +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.0/install.sh | bash +# load nvm in non-interactive shells +RUN echo 'export NVM_DIR="$HOME/.nvm"' > ~/.bashrc && \ + echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"' >> ~/.bashrc +# install node 10 +RUN bash -c '. ~/.bashrc; nvm install --lts=dubnium' && \ + ln -s "/root/.nvm/versions/node/$(ls -p /root/.nvm/versions/node | sort -Vr | head -1)bin/node" /usr/local/bin/node && \ + ln -s "/root/.nvm/versions/node/$(ls -p /root/.nvm/versions/node | sort -Vr | head -1)bin/npm" /usr/local/bin/npm +# setup Postgress +RUN su - postgres -c "/usr/bin/initdb" \ No newline at end of file diff --git a/.cicd/platforms/unpinned/centos-7.7-unpinned.dockerfile b/.cicd/platforms/unpinned/centos-7.7-unpinned.dockerfile index 91c243170b5..42f557efdf2 100644 --- a/.cicd/platforms/unpinned/centos-7.7-unpinned.dockerfile +++ b/.cicd/platforms/unpinned/centos-7.7-unpinned.dockerfile @@ -6,20 +6,20 @@ RUN yum update -y && \ yum --enablerepo=extras install -y centos-release-scl && \ yum --enablerepo=extras install -y devtoolset-8 && \ yum --enablerepo=extras install -y which git autoconf automake libtool make bzip2 doxygen \ - graphviz bzip2-devel openssl-devel gmp-devel ocaml libicu-devel \ - python python-devel rh-python36 gettext-devel file libusbx-devel \ - libcurl-devel patch vim-common jq llvm-toolset-7.0-llvm-devel llvm-toolset-7.0-llvm-static -# build cmake. -RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ + graphviz bzip2-devel openssl-devel gmp-devel ocaml \ + python python-devel rh-python36 file libusbx-devel \ + libcurl-devel patch vim-common jq llvm-toolset-7.0-llvm-devel llvm-toolset-7.0-llvm-static \ + glibc-locale-source glibc-langpack-en && \ + yum clean all && rm -rf /var/cache/yum +# build cmake +RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ + tar -xzf cmake-3.16.2.tar.gz && \ + cd cmake-3.16.2 && \ source /opt/rh/devtoolset-8/enable && \ - source /opt/rh/rh-python36/enable && \ - tar -xzf cmake-3.13.2.tar.gz && \ - cd cmake-3.13.2 && \ ./bootstrap --prefix=/usr/local && \ make -j$(nproc) && \ make install && \ - cd / && \ - rm -rf cmake-3.13.2.tar.gz /cmake-3.13.2 + rm -rf cmake-3.16.2.tar.gz cmake-3.16.2 # build boost RUN curl -LO https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.bz2 && \ source /opt/rh/devtoolset-8/enable && \ @@ -30,36 +30,31 @@ RUN curl -LO https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/bo ./b2 --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install && \ cd / && \ rm -rf boost_1_71_0.tar.bz2 /boost_1_71_0 -# build mongodb -RUN curl -LO https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-amazon-3.6.3.tgz && \ - tar -xzf mongodb-linux-x86_64-amazon-3.6.3.tgz && \ - rm -f mongodb-linux-x86_64-amazon-3.6.3.tgz -# build mongodb c driver -RUN curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz && \ +#install libpq postgresql-server +RUN yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm && \ + yum install -y postgresql13-devel postgresql13-server && \ + yum clean all && rm -rf /var/cache/yum +ENV PostgreSQL_ROOT=/usr/pgsql-13 +ENV PKG_CONFIG_PATH=/usr/pgsql-13/lib/pkgconfig:/usr/local/lib64/pkgconfig +#build libpqxx +RUN curl -L https://github.com/jtv/libpqxx/archive/7.2.1.tar.gz | tar zxvf - && \ + cd libpqxx-7.2.1 && \ source /opt/rh/devtoolset-8/enable && \ - source /opt/rh/rh-python36/enable && \ - tar -xzf mongo-c-driver-1.13.0.tar.gz && \ - cd mongo-c-driver-1.13.0 && \ - mkdir -p build && \ - cd build && \ - cmake --DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_BSON=ON -DENABLE_SSL=OPENSSL -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON -DENABLE_ICU=OFF -DENABLE_SNAPPY=OFF .. && \ - make -j$(nproc) && \ - make install && \ - cd / && \ - rm -rf mongo-c-driver-1.13.0.tar.gz /mongo-c-driver-1.13.0 -# build mongodb cxx driver -RUN curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz && \ - source /opt/rh/devtoolset-8/enable && \ - source /opt/rh/rh-python36/enable && \ - tar -xzf mongo-cxx-driver-r3.4.0.tar.gz && \ - cd mongo-cxx-driver-r3.4.0 && \ - sed -i 's/\"maxAwaitTimeMS\", ount/\"maxAwaitTimeMS\", static_cast(count)/' src/mongocxx/options/change_stream.cpp && \ - sed -i 's/add_subdirectory(test)//' src/mongocxx/CMakeLists.txt src/bsoncxx/CMakeLists.txt && \ - cd build && \ - cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. && \ - make -j$(nproc) && \ - make install && \ - cd / && \ - rm -rf mongo-cxx-driver-r3.4.0.tar.gz /mongo-cxx-driver-r3.4.0 -# add mongodb to path -ENV PATH=${PATH}:/mongodb-linux-x86_64-amazon-3.6.3/bin \ No newline at end of file + cmake -DSKIP_BUILD_TEST=ON -DCMAKE_BUILD_TYPE=Release -S . -B build && \ + cmake --build build && cmake --install build && \ + cd .. && rm -rf libpqxx-7.2.1 +# install nvm +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.0/install.sh | bash +# load nvm in non-interactive shells +RUN cp ~/.bashrc ~/.bashrc.bak && \ + cat ~/.bashrc.bak | tail -3 > ~/.bashrc && \ + cat ~/.bashrc.bak | head -n '-3' >> ~/.bashrc && \ + rm ~/.bashrc.bak +# install node 10 +RUN bash -c '. ~/.bashrc; nvm install --lts=dubnium' && \ + ln -s "/root/.nvm/versions/node/$(ls -p /root/.nvm/versions/node | sort -Vr | head -1)bin/node" /usr/local/bin/node +RUN yum install -y nodejs && \ + yum clean all && rm -rf /var/cache/yum +# setup Postgress +RUN localedef -c -f UTF-8 -i en_US en_US.UTF-8 && \ + su - postgres -c "/usr/pgsql-13/bin/initdb" \ No newline at end of file diff --git a/.cicd/platforms/unpinned/centos-8-unpinned.dockerfile b/.cicd/platforms/unpinned/centos-8-unpinned.dockerfile new file mode 100644 index 00000000000..e52a324cd2e --- /dev/null +++ b/.cicd/platforms/unpinned/centos-8-unpinned.dockerfile @@ -0,0 +1,59 @@ +FROM centos:8 +ENV VERSION 1 +#install dependencies +RUN yum update -y && \ + yum install -y epel-release && \ + yum --enablerepo=extras install -y which git autoconf automake libtool make bzip2 && \ + yum --enablerepo=extras install -y graphviz bzip2-devel openssl-devel gmp-devel && \ + yum --enablerepo=extras install -y file libusbx-devel && \ + yum --enablerepo=extras install -y libcurl-devel patch vim-common jq && \ + yum install -y python3 python3-devel clang llvm-devel llvm-static procps-ng util-linux sudo libstdc++ \ + glibc-locale-source glibc-langpack-en && \ + yum clean all && rm -rf /var/cache/yum +RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \ + dnf group install -y "Development Tools" && \ + dnf --enablerepo=powertools install -y doxygen ocaml +# cmake3.18.0 +RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.18.0/cmake-3.18.0.tar.gz && \ + tar -xzf cmake-3.18.0.tar.gz && \ + cd cmake-3.18.0 && \ + ./bootstrap --prefix=/usr/local && \ + make -j$(nproc) && make install && \ + rm -rf cmake-3.18.0.tar.gz cmake-3.18.2 +# build boost +RUN curl -LO https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.bz2 && \ + tar -xjf boost_1_72_0.tar.bz2 && \ + cd boost_1_72_0 && \ + ./bootstrap.sh --prefix=/usr/local && \ + ./b2 --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install && \ + cd / && \ + rm -rf boost_1_72_0.tar.bz2 /boost_1_72_0 +# install libpq & postgres +RUN dnf install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm && \ + dnf -qy module disable postgresql && \ + dnf install -y postgresql13-devel postgresql13-server \ + && dnf clean all && rm -rf /var/cache/yum +ENV PostgreSQL_ROOT=/usr/pgsql-13 +ENV PKG_CONFIG_PATH=/usr/pgsql-13/lib/pkgconfig:/usr/local/lib64/pkgconfig +#build libpqxx +RUN curl -L https://github.com/jtv/libpqxx/archive/7.2.1.tar.gz | tar zxvf - && \ + cd libpqxx-7.2.1 && \ + cmake -DSKIP_BUILD_TEST=ON -DCMAKE_BUILD_TYPE=Release -S . -B build && \ + cmake --build build && cmake --install build && \ + cd .. && rm -rf libpqxx-7.2.1 +# install nvm +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.0/install.sh | bash +# load nvm in non-interactive shells +RUN cp ~/.bashrc ~/.bashrc.bak && \ + cat ~/.bashrc.bak | tail -3 > ~/.bashrc && \ + cat ~/.bashrc.bak | head -n '-3' >> ~/.bashrc && \ + rm ~/.bashrc.bak +# install node 10 +RUN bash -c '. ~/.bashrc; nvm install --lts=dubnium' && \ + ln -s "/root/.nvm/versions/node/$(ls -p /root/.nvm/versions/node | sort -Vr | head -1)bin/node" /usr/local/bin/node +RUN yum install -y nodejs && \ + yum clean all && rm -rf /var/cache/yum +RUN ln -s /usr/lib64/libtinfo.so.6 /usr/local/lib/libtinfo.so +# setup Postgress +RUN localedef -c -f UTF-8 -i en_US en_US.UTF-8 && \ + su - postgres -c "/usr/pgsql-13/bin/initdb" \ No newline at end of file diff --git a/.cicd/platforms/unpinned/macos-10.14-unpinned.sh b/.cicd/platforms/unpinned/macos-10.14-unpinned.sh index 79992322746..08fcdea7f54 100755 --- a/.cicd/platforms/unpinned/macos-10.14-unpinned.sh +++ b/.cicd/platforms/unpinned/macos-10.14-unpinned.sh @@ -2,34 +2,12 @@ set -eo pipefail VERSION=1 brew update -brew install git icu4c python libtool libusb graphviz automake wget gmp llvm@7 pkgconfig doxygen openssl@1.1 cmake jq || : -curl -LO https://raw.githubusercontent.com/Homebrew/homebrew-core/0b9ee3127763e8ebce62599bae85f37ab1687622/Formula/boost.rb -brew install -f boost.rb -# install mongoDB -cd ~ -curl -OL https://fastdl.mongodb.org/osx/mongodb-osx-ssl-x86_64-3.6.3.tgz -tar -xzf mongodb-osx-ssl-x86_64-3.6.3.tgz -rm -f mongodb-osx-ssl-x86_64-3.6.3.tgz -ln -s ~/mongodb-osx-x86_64-3.6.3 ~/mongodb -# install mongo-c-driver from source -cd /tmp -curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz -tar -xzf mongo-c-driver-1.13.0.tar.gz -cd mongo-c-driver-1.13.0 -mkdir -p cmake-build -cd cmake-build -cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX='/usr/local' -DENABLE_BSON=ON -DENABLE_SSL=DARWIN -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON -DENABLE_ICU=OFF -DENABLE_SASL=OFF -DENABLE_SNAPPY=OFF .. -make -j $(getconf _NPROCESSORS_ONLN) -sudo make install -cd ../.. -rm mongo-c-driver-1.13.0.tar.gz -# install mongo-cxx-driver from source -cd /tmp -curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz -tar -xzf mongo-cxx-driver-r3.4.0.tar.gz -cd mongo-cxx-driver-r3.4.0/build -cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX='/usr/local' .. -make -j $(getconf _NPROCESSORS_ONLN) VERBOSE=1 -sudo make install -cd ../.. -rm -f mongo-cxx-driver-r3.4.0.tar.gz \ No newline at end of file +brew install git cmake python libtool libusb graphviz automake wget gmp pkgconfig doxygen openssl@1.1 jq boost libpq postgres || : +# libpqxx 7.3+ installations on mojave try to import libs not present in the sdk. pin to libpqxx 7.2.1 instead. +curl -LO https://raw.githubusercontent.com/Homebrew/homebrew-core/d14398187084e1d3fd1763ec13cea1044946a51f/Formula/libpqxx.rb +brew install -f ./libpqxx.rb +# install nvm for ship_test +cd ~ && brew install nvm && mkdir -p ~/.nvm && echo "export NVM_DIR=$HOME/.nvm" >> ~/.bash_profile && echo 'source $(brew --prefix nvm)/nvm.sh' >> ~/.bash_profile && cat ~/.bash_profile && source ~/.bash_profile && echo $NVM_DIR && nvm install --lts=dubnium +# initialize postgres configuration files +sudo rm -rf /usr/local/var/postgres +initdb --locale=C -E UTF-8 /usr/local/var/postgres diff --git a/.cicd/platforms/unpinned/macos-10.15-unpinned.sh b/.cicd/platforms/unpinned/macos-10.15-unpinned.sh new file mode 100755 index 00000000000..b1b79078349 --- /dev/null +++ b/.cicd/platforms/unpinned/macos-10.15-unpinned.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -eo pipefail +VERSION=1 +export SDKROOT="$(xcrun --sdk macosx --show-sdk-path)" +brew update +brew install git cmake python libtool libusb graphviz automake wget gmp pkgconfig doxygen openssl jq boost libpq libpqxx postgres || : +# install nvm for ship_test +cd ~ && brew install nvm && mkdir -p ~/.nvm && echo "export NVM_DIR=$HOME/.nvm" >> ~/.bash_profile && echo 'source $(brew --prefix nvm)/nvm.sh' >> ~/.bash_profile && cat ~/.bash_profile && source ~/.bash_profile && echo $NVM_DIR && nvm install --lts=dubnium +# initialize postgres configuration files +sudo rm -rf /usr/local/var/postgres +initdb --locale=C -E UTF-8 /usr/local/var/postgres diff --git a/.cicd/platforms/unpinned/ubuntu-18.04-unpinned.dockerfile b/.cicd/platforms/unpinned/ubuntu-18.04-unpinned.dockerfile index 36902c23f19..813549aef17 100644 --- a/.cicd/platforms/unpinned/ubuntu-18.04-unpinned.dockerfile +++ b/.cicd/platforms/unpinned/ubuntu-18.04-unpinned.dockerfile @@ -5,18 +5,19 @@ RUN apt-get update && \ apt-get upgrade -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y git make \ bzip2 automake libbz2-dev libssl-dev doxygen graphviz libgmp3-dev \ - autotools-dev libicu-dev python2.7 python2.7-dev python3 python3-dev \ + autotools-dev python2.7 python2.7-dev python3 python3-dev \ autoconf libtool curl zlib1g-dev sudo ruby libusb-1.0-0-dev \ - libcurl4-gnutls-dev pkg-config patch llvm-7-dev clang-7 vim-common jq -# build cmake. -RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ - tar -xzf cmake-3.13.2.tar.gz && \ - cd cmake-3.13.2 && \ + libcurl4-gnutls-dev pkg-config patch llvm-7-dev clang-7 vim-common jq && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* +# build cmake +RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ + tar -xzf cmake-3.16.2.tar.gz && \ + cd cmake-3.16.2 && \ ./bootstrap --prefix=/usr/local && \ make -j$(nproc) && \ make install && \ - cd / && \ - rm -rf cmake-3.13.2.tar.gz /cmake-3.13.2 + rm -rf cmake-3.16.2.tar.gz cmake-3.16.2 # build boost RUN curl -LO https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.bz2 && \ tar -xjf boost_1_71_0.tar.bz2 && \ @@ -25,32 +26,31 @@ RUN curl -LO https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/bo ./b2 --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -j$(nproc) install && \ cd / && \ rm -rf boost_1_71_0.tar.bz2 /boost_1_71_0 -# build mongodb -RUN curl -LO http://downloads.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1804-4.1.1.tgz && \ - tar -xzf mongodb-linux-x86_64-ubuntu1804-4.1.1.tgz && \ - rm -f mongodb-linux-x86_64-ubuntu1804-4.1.1.tgz -# build mongodb c driver -RUN curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz && \ - tar -xzf mongo-c-driver-1.13.0.tar.gz && \ - cd mongo-c-driver-1.13.0 && \ - mkdir -p build && \ - cd build && \ - cmake --DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_BSON=ON -DENABLE_SSL=OPENSSL -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON .. && \ - make -j$(nproc) && \ - make install && \ - cd / && \ - rm -rf mongo-c-driver-1.13.0.tar.gz /mongo-c-driver-1.13.0 -# build mongodb cxx driver -RUN curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz && \ - tar -xzf mongo-cxx-driver-r3.4.0.tar.gz && \ - cd mongo-cxx-driver-r3.4.0 && \ - sed -i 's/\"maxAwaitTimeMS\", count/\"maxAwaitTimeMS\", static_cast(count)/' src/mongocxx/options/change_stream.cpp && \ - sed -i 's/add_subdirectory(test)//' src/mongocxx/CMakeLists.txt src/bsoncxx/CMakeLists.txt && \ - cd build && \ - cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. && \ - make -j$(nproc) && \ - make install && \ - cd / && \ - rm -rf mongo-cxx-driver-r3.4.0.tar.gz /mongo-cxx-driver-r3.4.0 -# add mongodb to path -ENV PATH=${PATH}:/mongodb-linux-x86_64-ubuntu1804-4.1.1/bin \ No newline at end of file +# install libpq, postgresql-13 +ENV TZ=America/Chicago +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \ + echo "deb http://apt.postgresql.org/pub/repos/apt bionic-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \ + curl -sL https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \ + apt-get update && apt-get -y install libpq-dev postgresql-13 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* +#build libpqxx +RUN curl -L https://github.com/jtv/libpqxx/archive/7.2.1.tar.gz | tar zxvf - && \ + cd libpqxx-7.2.1 && \ + cmake -DSKIP_BUILD_TEST=ON -DPostgreSQL_TYPE_INCLUDE_DIR=/usr/include/postgresql -DCMAKE_BUILD_TYPE=Release -S . -B build && \ + cmake --build build && cmake --install build && \ + cd .. && rm -rf libpqxx-7.2.1 +# install nvm +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.0/install.sh | bash +# load nvm in non-interactive shells +RUN cp ~/.bashrc ~/.bashrc.bak && \ + cat ~/.bashrc.bak | tail -3 > ~/.bashrc && \ + cat ~/.bashrc.bak | head -n '-3' >> ~/.bashrc && \ + rm ~/.bashrc.bak +# install node 10 +RUN bash -c '. ~/.bashrc; nvm install --lts=dubnium' && \ + ln -s "/root/.nvm/versions/node/$(ls -p /root/.nvm/versions/node | sort -Vr | head -1)bin/node" /usr/local/bin/node +RUN curl -sL https://deb.nodesource.com/setup_13.x | sudo -E bash - +RUN sudo apt-get install -y nodejs && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* diff --git a/.cicd/platforms/unpinned/ubuntu-20.04-unpinned.dockerfile b/.cicd/platforms/unpinned/ubuntu-20.04-unpinned.dockerfile new file mode 100644 index 00000000000..384afc18868 --- /dev/null +++ b/.cicd/platforms/unpinned/ubuntu-20.04-unpinned.dockerfile @@ -0,0 +1,52 @@ +FROM ubuntu:20.04 +ENV VERSION 1 +# install dependencies. +RUN apt-get update && \ + apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y git make \ + bzip2 automake libbz2-dev libssl-dev doxygen graphviz libgmp3-dev \ + autotools-dev python2.7 python2.7-dev python3 python3-dev \ + autoconf libtool curl zlib1g-dev sudo ruby libusb-1.0-0-dev \ + libcurl4-gnutls-dev pkg-config patch llvm-7-dev clang-7 vim-common jq g++ gnupg && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# build cmake +RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2.tar.gz && \ + tar -xzf cmake-3.16.2.tar.gz && \ + cd cmake-3.16.2 && \ + ./bootstrap --prefix=/usr/local && \ + make -j$(nproc) && \ + make install && \ + rm -rf cmake-3.16.2.tar.gz cmake-3.16.2 +# build boost +RUN curl -LO https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.bz2 && \ + tar -xjf boost_1_71_0.tar.bz2 && \ + cd boost_1_71_0 && \ + ./bootstrap.sh --prefix=/usr/local && \ + ./b2 --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -j$(nproc) install && \ + cd / && \ + rm -rf boost_1_71_0.tar.bz2 /boost_1_71_0 +# install libpq postgresql +ENV TZ=America/Chicago +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \ + echo "deb http://apt.postgresql.org/pub/repos/apt focal-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \ + curl -sL https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \ + apt-get update && apt-get -y install libpq-dev postgresql-13 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* +#build libpqxx +RUN curl -L https://github.com/jtv/libpqxx/archive/7.2.1.tar.gz | tar zxvf - && \ + cd libpqxx-7.2.1 && \ + cmake -DSKIP_BUILD_TEST=ON -DPostgreSQL_TYPE_INCLUDE_DIR=/usr/include/postgresql -DCMAKE_BUILD_TYPE=Release -S . -B build && \ + cmake --build build && cmake --install build && \ + cd .. && rm -rf libpqxx-7.2.1 +# install node 12 +RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \ + . /etc/lsb-release && \ + echo "deb https://deb.nodesource.com/node_12.x $DISTRIB_CODENAME main" | tee /etc/apt/sources.list.d/nodesource.list && \ + echo "deb-src https://deb.nodesource.com/node_12.x $DISTRIB_CODENAME main" | tee -a /etc/apt/sources.list.d/nodesource.list && \ + apt-get update && \ + apt-get install -y nodejs && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* diff --git a/.cicd/submodule-regression-check.sh b/.cicd/submodule-regression-check.sh index 35519a8a5c4..43e5af69803 100755 --- a/.cicd/submodule-regression-check.sh +++ b/.cicd/submodule-regression-check.sh @@ -19,15 +19,21 @@ while read -r a b; do done < <(git submodule --quiet foreach --recursive 'echo $path `git log -1 --format=%ct`') echo "getting submodule info for $BASE_BRANCH" -git checkout $BASE_BRANCH 1> /dev/null -git submodule update --init 1> /dev/null +GIT_CHECKOUT="git checkout '$BASE_BRANCH' 1> /dev/null" +echo "$ $GIT_CHECKOUT" +eval $GIT_CHECKOUT +GIT_SUBMODULE="git submodule update --init 1> /dev/null" +echo "$ $GIT_SUBMODULE" +eval $GIT_SUBMODULE + while read -r a b; do BASE_MAP[$a]=$b done < <(git submodule --quiet foreach --recursive 'echo $path `git log -1 --format=%ct`') echo "switching back to $CURRENT_BRANCH..." -echo "git checkout -qf $CURRENT_BRANCH" -git checkout -qf $CURRENT_BRANCH 1> /dev/null +GIT_CHECKOUT="git checkout -qf '$CURRENT_BRANCH' 1> /dev/null" +echo "$ $GIT_CHECKOUT" +eval $GIT_CHECKOUT for k in "${!BASE_MAP[@]}"; do base_ts=${BASE_MAP[$k]} @@ -37,11 +43,12 @@ for k in "${!BASE_MAP[@]}"; do echo " timestamp on $BASE_BRANCH: $base_ts" if (( $pr_ts < $base_ts)); then echo "$k is older on $CURRENT_BRANCH than $BASE_BRANCH; investigating the difference between $CURRENT_BRANCH and $BASE_BRANCH to look for $k changing..." - if [[ ! -z $(for c in $(git --no-pager log $CURRENT_BRANCH ^$BASE_BRANCH --pretty=format:"%H"); do git show --pretty="" --name-only $c; done | grep "^$k$") ]]; then + GIT_LOG="git --no-pager log '$CURRENT_BRANCH' '^$BASE_BRANCH' --pretty=format:\"%H\"" + if [[ ! -z $(for c in $(eval $GIT_LOG); do git show --pretty="" --name-only $c; done | grep "^$k$") ]]; then echo "ERROR: $k has regressed" exit 1 else echo "$k was not in the diff; no regression detected" fi fi -done \ No newline at end of file +done diff --git a/.cicd/test.sh b/.cicd/test.sh index 44b46e7b133..e76881c20c1 100755 --- a/.cicd/test.sh +++ b/.cicd/test.sh @@ -4,8 +4,8 @@ set -eo pipefail . ./.cicd/helpers/general.sh # tests if [[ $(uname) == 'Darwin' ]]; then # macOS - export PATH=$PATH:~/mongodb/bin set +e # defer error handling to end + [[ "$CI" == 'true' ]] && source ~/.bash_profile TEST_COMMAND="\"./$1\" ${@: 2}" echo "$ $TEST_COMMAND" eval $TEST_COMMAND @@ -26,6 +26,11 @@ if [[ "$BUILDKITE" == 'true' ]]; then cd build # upload artifacts echo '--- :arrow_up: Uploading Artifacts' + echo 'Compressing configuration' + [[ -d etc ]] && tar czf etc.tar.gz etc + echo 'Compressing logs' + [[ -d var ]] && tar czf var.tar.gz var + [[ -d eosio-ignition-wd ]] && tar czf eosio-ignition-wd.tar.gz eosio-ignition-wd echo 'Compressing core dumps...' [[ $((`ls -1 core.* 2>/dev/null | wc -l`)) != 0 ]] && tar czf core.tar.gz core.* || : # collect core dumps echo 'Exporting xUnit XML' @@ -34,7 +39,11 @@ if [[ "$BUILDKITE" == 'true' ]]; then [[ -f config.ini ]] && buildkite-agent artifact upload config.ini [[ -f core.tar.gz ]] && buildkite-agent artifact upload core.tar.gz [[ -f genesis.json ]] && buildkite-agent artifact upload genesis.json - [[ -f mongod.log ]] && buildkite-agent artifact upload mongod.log + [[ -f etc.tar.gz ]] && buildkite-agent artifact upload etc.tar.gz + [[ -f ctest-output.log ]] && buildkite-agent artifact upload ctest-output.log + [[ -f var.tar.gz ]] && buildkite-agent artifact upload var.tar.gz + [[ -f eosio-ignition-wd.tar.gz ]] && buildkite-agent artifact upload eosio-ignition-wd.tar.gz + [[ -f bios_boot.sh ]] && buildkite-agent artifact upload bios_boot.sh buildkite-agent artifact upload test-results.xml echo 'Done uploading artifacts.' fi diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 20c04810660..7e70d4f1b9f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -415,7 +415,7 @@ jobs: git submodule update --init --force --recursive - name: Build run: | - ./.cicd/platforms/unpinned/macos-10.14-unpinned.sh + ./.cicd/platforms/unpinned/macos-10.15-unpinned.sh ./.cicd/build.sh tar -pczf build.tar.gz build - name: Upload Build Artifact @@ -441,7 +441,7 @@ jobs: name: macos-1015-build - name: Parallel Test run: | - ./.cicd/platforms/unpinned/macos-10.14-unpinned.sh + ./.cicd/platforms/unpinned/macos-10.15-unpinned.sh tar -xzf macos-1015-build/build.tar.gz ./.cicd/test.sh scripts/parallel-test.sh macos-1015-wasm-test: @@ -462,7 +462,7 @@ jobs: name: macos-1015-build - name: WASM Spec Test run: | - ./.cicd/platforms/unpinned/macos-10.14-unpinned.sh + ./.cicd/platforms/unpinned/macos-10.15-unpinned.sh tar -xzf macos-1015-build/build.tar.gz ./.cicd/test.sh scripts/wasm-spec-test.sh macos-1015-serial-test: @@ -483,6 +483,6 @@ jobs: name: macos-1015-build - name: Serial Test run: | - ./.cicd/platforms/unpinned/macos-10.14-unpinned.sh + ./.cicd/platforms/unpinned/macos-10.15-unpinned.sh tar -xzf macos-1015-build/build.tar.gz - ./.cicd/test.sh scripts/serial-test.sh \ No newline at end of file + ./.cicd/test.sh scripts/serial-test.sh diff --git a/.gitignore b/.gitignore index 4c6f4f1a0f8..8cd0213e1d7 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ Makefile compile_commands.json moc_* *.moc +.clangd/ genesis.json hardfork.hpp diff --git a/.gitmodules b/.gitmodules index 806b2bcd783..f3d406ce8f4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,20 +1,6 @@ -[submodule "libraries/chainbase"] - path = libraries/chainbase - url = https://github.com/eosio/chainbase - ignore = dirty -[submodule "libraries/appbase"] - path = libraries/appbase - url = https://github.com/eosio/appbase - ignore = dirty [submodule "libraries/softfloat"] path = libraries/softfloat url = https://github.com/eosio/berkeley-softfloat-3 -[submodule "libraries/fc"] - path = libraries/fc - url = https://github.com/EOSIO/fc -[submodule "libraries/wabt"] - path = libraries/wabt - url = https://github.com/EOSIO/wabt [submodule "libraries/yubihsm"] path = libraries/yubihsm url = https://github.com/Yubico/yubihsm-shell @@ -24,3 +10,21 @@ [submodule "eosio-wasm-spec-tests"] path = eosio-wasm-spec-tests url = https://github.com/EOSIO/eosio-wasm-spec-tests +[submodule "libraries/abieos"] + path = libraries/abieos + url = https://github.com/EOSIO/abieos.git +[submodule "libraries/rocksdb"] + path = libraries/rocksdb + url = https://github.com/facebook/rocksdb.git +[submodule "libraries/amqp-cpp"] + path = libraries/amqp-cpp + url = https://github.com/CopernicaMarketingSoftware/AMQP-CPP +[submodule "libraries/fc"] + path = libraries/fc + url = https://github.com/eosio/fc +[submodule "libraries/chainbase"] + path = libraries/chainbase + url = https://github.com/eosio/chainbase +[submodule "libraries/appbase"] + path = libraries/appbase + url = https://github.com/eosio/appbase diff --git a/CMakeLists.txt b/CMakeLists.txt index f5396a82adf..546213e1f5c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,10 +10,6 @@ endif() list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules") -if (UNIX AND APPLE) - list(APPEND CMAKE_PREFIX_PATH "/usr/local/opt/gettext") -endif() - include( GNUInstallDirs ) include( InstallDirectoryPermissions ) include( MASSigning ) @@ -24,8 +20,8 @@ set( CMAKE_CXX_EXTENSIONS ON ) set( CXX_STANDARD_REQUIRED ON) set(VERSION_MAJOR 2) -set(VERSION_MINOR 0) -set(VERSION_PATCH 12) +set(VERSION_MINOR 1) +set(VERSION_PATCH 0) #set(VERSION_SUFFIX rc3) if(VERSION_SUFFIX) @@ -37,6 +33,8 @@ endif() set( CLI_CLIENT_EXECUTABLE_NAME cleos ) set( NODE_EXECUTABLE_NAME nodeos ) set( KEY_STORE_EXECUTABLE_NAME keosd ) +set( RODEOS_EXECUTABLE_NAME rodeos ) +set( TESTER_EXECUTABLE_NAME eosio-tester ) # http://stackoverflow.com/a/18369825 if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") @@ -47,15 +45,18 @@ elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID} if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0) message(FATAL_ERROR "Clang version must be at least 5.0!") endif() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wsign-compare -Wrange-loop-analysis") +endif() + +if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") + set(CMAKE_VISIBILITY_INLINES_HIDDEN ON) endif() if ("${CMAKE_GENERATOR}" STREQUAL "Ninja") add_compile_options(-fdiagnostics-color=always) endif() -set(CMAKE_EXPORT_COMPILE_COMMANDS "ON") set(BUILD_DOXYGEN FALSE CACHE BOOL "Build doxygen documentation on every make") -set(BUILD_MONGO_DB_PLUGIN FALSE CACHE BOOL "Build mongo database plugin") set(ENABLE_MULTIVERSION_PROTOCOL_TEST FALSE CACHE BOOL "Enable nodeos multiversion protocol test") # add defaults for openssl @@ -66,16 +67,16 @@ endif() # WASM runtimes to enable. Each runtime in this list will have: # * definition EOSIO__RUNTIME_ENABLED defined in public libchain interface # * ctest entries with --runtime -list(APPEND EOSIO_WASM_RUNTIMES wabt) #always enable wabt; it works everywhere and parts of eosio still assume it's always available +# TODO reenable later if(CMAKE_SIZEOF_VOID_P EQUAL 8 AND NOT WIN32) if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux" AND "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "x86_64") list(APPEND EOSIO_WASM_RUNTIMES eos-vm-oc) # EOS VM OC requires LLVM, but move the check up here to a central location so that the EosioTester.cmakes # can be created with the exact version found find_package(LLVM REQUIRED CONFIG) - if(LLVM_VERSION_MAJOR VERSION_LESS 7 OR LLVM_VERSION_MAJOR VERSION_GREATER_EQUAL 12) - message(FATAL_ERROR "EOSIO requires an LLVM version 7 through 11") - endif() + if(LLVM_VERSION_MAJOR VERSION_LESS 7 OR LLVM_VERSION_MAJOR VERSION_GREATER_EQUAL 12) + message(FATAL_ERROR "EOSIO requires an LLVM version 7 through 11") + endif() endif() endif() @@ -98,6 +99,7 @@ else() set(no_whole_archive_flag "--no-whole-archive") endif() +set(Boost_USE_MULTITHREADED ON) set( Boost_USE_STATIC_LIBS ON CACHE STRING "ON or OFF" ) # Most boost deps get implictly picked up via fc, as just about everything links to fc. In addition we pick up # the pthread dependency through fc. @@ -106,11 +108,13 @@ find_package(Boost 1.67 REQUIRED COMPONENTS program_options unit_test_framework) if( APPLE AND UNIX ) # Apple Specific Options Here message( STATUS "Configuring EOSIO on macOS" ) - set( CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_CXX_FLAGS} -Wall -Wno-deprecated-declarations" ) + set( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wno-deprecated-declarations" ) + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-deprecated-declarations" ) else() # Linux Specific Options Here message( STATUS "Configuring EOSIO on Linux" ) - set( CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_CXX_FLAGS} -Wall" ) + set( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall" ) + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall" ) if ( FULL_STATIC_BUILD ) set( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libstdc++ -static-libgcc") endif ( FULL_STATIC_BUILD ) @@ -122,6 +126,8 @@ else() endif() endif() +option(EOSIO_ENABLE_DEVELOPER_OPTIONS "enable developer options for EOSIO" OFF) + # based on http://www.delorie.com/gnu/docs/gdb/gdb_70.html # uncomment this line to tell GDB about macros (slows compile times) # set( CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -gdwarf-2 -g3" ) @@ -129,6 +135,7 @@ endif() set(ENABLE_COVERAGE_TESTING FALSE CACHE BOOL "Build EOSIO for code coverage analysis") if(ENABLE_COVERAGE_TESTING) + SET(CMAKE_C_FLAGS "--coverage ${CMAKE_C_FLAGS}") SET(CMAKE_CXX_FLAGS "--coverage ${CMAKE_CXX_FLAGS}") find_program( LCOV_PATH lcov ) find_program( LLVMCOV_PATH llvm-cov ) @@ -155,18 +162,34 @@ endif() message( STATUS "Using '${EOSIO_ROOT_KEY}' as public key for 'eosio' account" ) +find_package( Gperftools QUIET ) +if( GPERFTOOLS_FOUND ) + message( STATUS "Found gperftools; compiling EOSIO with TCMalloc") + #if doing this by the book, simply link_libraries( ${GPERFTOOLS_TCMALLOC} ) here. That will + #give the performance benefits of tcmalloc but since it won't be linked last + #the heap profiler & checker may not be accurate. This here is rather undocumented behavior + #to stuff a library toward the end of the link list + set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} ${GPERFTOOLS_TCMALLOC}") + set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} ${GPERFTOOLS_TCMALLOC}") +endif() + add_subdirectory( libraries ) add_subdirectory( plugins ) add_subdirectory( programs ) add_subdirectory( scripts ) add_subdirectory( unittests ) +add_subdirectory( contracts ) add_subdirectory( tests ) add_subdirectory( tools ) +option(DISABLE_WASM_SPEC_TESTS "disable building of wasm spec unit tests" OFF) + if (NOT DISABLE_WASM_SPEC_TESTS) add_subdirectory( eosio-wasm-spec-tests/generated-tests ) endif() +set(CMAKE_EXPORT_COMPILE_COMMANDS "ON") + install_directory_permissions(DIRECTORY ${CMAKE_INSTALL_FULL_SYSCONFDIR}/eosio) install(FILES testnet.template DESTINATION ${CMAKE_INSTALL_FULL_SYSCONFDIR}/eosio/launcher) @@ -174,8 +197,13 @@ install_directory_permissions(DIRECTORY ${CMAKE_INSTALL_FULL_SYSCONFDIR}/eosio) install_directory_permissions(DIRECTORY ${CMAKE_INSTALL_FULL_SYSCONFDIR}/eosio/launcher) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/testnet.template ${CMAKE_CURRENT_BINARY_DIR}/etc/eosio/launcher/testnet.template COPYONLY) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/eosio.version.in ${CMAKE_CURRENT_BINARY_DIR}/eosio.version.hpp) -install(FILES ${CMAKE_CURRENT_BINARY_DIR}/eosio.version.hpp DESTINATION ${CMAKE_INSTALL_FULL_INCLUDEDIR}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/contracts/contracts/eosio.bios/eosio.bios.abi DESTINATION ${CMAKE_INSTALL_FULL_SYSCONFDIR}/eosio/contracts) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/contracts/contracts/eosio.bios/eosio.bios.wasm DESTINATION ${CMAKE_INSTALL_FULL_SYSCONFDIR}/eosio/contracts) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/contracts/contracts/eosio.boot/eosio.boot.abi DESTINATION ${CMAKE_INSTALL_FULL_SYSCONFDIR}/eosio/contracts) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/contracts/contracts/eosio.boot/eosio.boot.wasm DESTINATION ${CMAKE_INSTALL_FULL_SYSCONFDIR}/eosio/contracts) + +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/version.in ${CMAKE_CURRENT_BINARY_DIR}/version.hpp) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.hpp DESTINATION ${CMAKE_INSTALL_FULL_INCLUDEDIR}/eosio) set(EOS_ROOT_DIR ${CMAKE_BINARY_DIR}) configure_file(${CMAKE_SOURCE_DIR}/CMakeModules/eosio-config.cmake.in ${CMAKE_BINARY_DIR}/lib/cmake/eosio/eosio-config.cmake @ONLY) @@ -189,8 +217,6 @@ install(FILES ${CMAKE_BINARY_DIR}/modules/EosioTester.cmake DESTINATION ${CMAKE_ configure_file(${CMAKE_SOURCE_DIR}/LICENSE ${CMAKE_BINARY_DIR}/licenses/eosio/LICENSE COPYONLY) -configure_file(${CMAKE_SOURCE_DIR}/libraries/wabt/LICENSE - ${CMAKE_BINARY_DIR}/licenses/eosio/LICENSE.wabt COPYONLY) configure_file(${CMAKE_SOURCE_DIR}/libraries/softfloat/COPYING.txt ${CMAKE_BINARY_DIR}/licenses/eosio/LICENSE.softfloat COPYONLY) configure_file(${CMAKE_SOURCE_DIR}/libraries/wasm-jit/LICENSE @@ -205,9 +231,16 @@ configure_file(${CMAKE_SOURCE_DIR}/libraries/yubihsm/LICENSE ${CMAKE_BINARY_DIR}/licenses/eosio/LICENSE.yubihsm COPYONLY) configure_file(${CMAKE_SOURCE_DIR}/libraries/eos-vm/LICENSE ${CMAKE_BINARY_DIR}/licenses/eosio/LICENSE.eos-vm COPYONLY) +configure_file(${CMAKE_SOURCE_DIR}/libraries/rocksdb/LICENSE.Apache + ${CMAKE_BINARY_DIR}/licenses/eosio/LICENSE.rocksdb COPYONLY) +configure_file(${CMAKE_SOURCE_DIR}/libraries/rocksdb/LICENSE.leveldb + ${CMAKE_BINARY_DIR}/licenses/eosio/LICENSE.leveldb COPYONLY) +configure_file(${CMAKE_SOURCE_DIR}/libraries/amqp-cpp/LICENSE + ${CMAKE_BINARY_DIR}/licenses/eosio/LICENSE.amqpcpp COPYONLY) +file(DOWNLOAD https://raw.githubusercontent.com/jtv/libpqxx/master/COPYING + ${CMAKE_BINARY_DIR}/licenses/eosio/LICENSE.libpqxx) install(FILES LICENSE DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ COMPONENT base) -install(FILES libraries/wabt/LICENSE DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ RENAME LICENSE.wabt COMPONENT base) install(FILES libraries/softfloat/COPYING.txt DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ RENAME LICENSE.softfloat COMPONENT base) install(FILES libraries/wasm-jit/LICENSE DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ RENAME LICENSE.wavm COMPONENT base) install(FILES libraries/fc/secp256k1/upstream/COPYING DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ RENAME LICENSE.secp256k1 COMPONENT base) @@ -215,6 +248,10 @@ install(FILES libraries/fc/include/fc/crypto/webauthn_json/license.txt DESTINATI install(FILES libraries/fc/src/network/LICENSE.go DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ COMPONENT base) install(FILES libraries/yubihsm/LICENSE DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ RENAME LICENSE.yubihsm COMPONENT base) install(FILES libraries/eos-vm/LICENSE DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ RENAME LICENSE.eos-vm COMPONENT base) +install(FILES libraries/rocksdb/LICENSE.Apache DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ RENAME LICENSE.rocksdb COMPONENT base) +install(FILES libraries/rocksdb/LICENSE.leveldb DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ RENAME LICENSE.leveldb COMPONENT base) +install(FILES libraries/amqp-cpp/LICENSE DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ RENAME LICENSE.amqpcpp COMPONENT base) +install(FILES ${CMAKE_BINARY_DIR}/licenses/eosio/LICENSE.libpqxx DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ COMPONENT base) add_custom_target(base-install COMMAND "${CMAKE_COMMAND}" --build "${CMAKE_BINARY_DIR}" diff --git a/CMakeModules/EosioTester.cmake.in b/CMakeModules/EosioTester.cmake.in index 988fd241095..87e3fc60b03 100644 --- a/CMakeModules/EosioTester.cmake.in +++ b/CMakeModules/EosioTester.cmake.in @@ -30,6 +30,11 @@ set( CMAKE_CXX_STANDARD 17 ) set( CMAKE_CXX_EXTENSIONS ON ) set( CXX_STANDARD_REQUIRED ON ) +#adds -pthread. Ubuntu eosio.contracts build breaks without this flag specified +set(CMAKE_THREAD_PREFER_PTHREAD TRUE) +set(THREADS_PREFER_PTHREAD_FLAG TRUE) +find_package(Threads) + if ( APPLE ) set( CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_CXX_FLAGS} -Wall -Wno-deprecated-declarations" ) else ( APPLE ) @@ -39,7 +44,7 @@ endif ( APPLE ) ### Remove after Boost 1.70 CMake fixes are in place set( Boost_NO_BOOST_CMAKE ON CACHE STRING "ON or OFF" ) set( Boost_USE_STATIC_LIBS ON CACHE STRING "ON or OFF" ) -find_package(Boost 1.67 REQUIRED COMPONENTS +find_package(Boost @Boost_MAJOR_VERSION@.@Boost_MINOR_VERSION@ EXACT REQUIRED COMPONENTS date_time filesystem system @@ -47,6 +52,7 @@ find_package(Boost 1.67 REQUIRED COMPONENTS iostreams unit_test_framework) +find_library(librocksdb rocksdb @CMAKE_INSTALL_PREFIX@/lib64 @CMAKE_INSTALL_PREFIX@/lib) find_library(libtester eosio_testing @CMAKE_INSTALL_PREFIX@/lib64 @CMAKE_INSTALL_PREFIX@/lib NO_DEFAULT_PATH) find_library(libchain eosio_chain @CMAKE_INSTALL_PREFIX@/lib64 @CMAKE_INSTALL_PREFIX@/lib NO_DEFAULT_PATH) if ( "${CMAKE_BUILD_TYPE}" STREQUAL "Debug" ) @@ -59,9 +65,7 @@ endif() find_library(libwasm WASM @CMAKE_INSTALL_PREFIX@/lib64 @CMAKE_INSTALL_PREFIX@/lib NO_DEFAULT_PATH) find_library(libwast WAST @CMAKE_INSTALL_PREFIX@/lib64 @CMAKE_INSTALL_PREFIX@/lib NO_DEFAULT_PATH) -find_library(libwabt wabt @CMAKE_INSTALL_PREFIX@/lib64 @CMAKE_INSTALL_PREFIX@/lib NO_DEFAULT_PATH) find_library(libir IR @CMAKE_INSTALL_PREFIX@/lib64 @CMAKE_INSTALL_PREFIX@/lib NO_DEFAULT_PATH) -find_library(libplatform Platform @CMAKE_INSTALL_PREFIX@/lib64 @CMAKE_INSTALL_PREFIX@/lib NO_DEFAULT_PATH) find_library(liblogging Logging @CMAKE_INSTALL_PREFIX@/lib64 @CMAKE_INSTALL_PREFIX@/lib NO_DEFAULT_PATH) find_library(libruntime Runtime @CMAKE_INSTALL_PREFIX@/lib64 @CMAKE_INSTALL_PREFIX@/lib NO_DEFAULT_PATH) find_library(libsoftfloat softfloat @CMAKE_INSTALL_PREFIX@/lib64 @CMAKE_INSTALL_PREFIX@/lib NO_DEFAULT_PATH) @@ -71,12 +75,11 @@ get_filename_component(ssldir @OPENSSL_SSL_LIBRARY@ DIRECTORY) find_library(libosssl ssl "${ssldir}" NO_DEFAULT_PATH) find_library(libchainbase chainbase @CMAKE_INSTALL_PREFIX@/lib64 @CMAKE_INSTALL_PREFIX@/lib NO_DEFAULT_PATH) find_library(libbuiltins builtins @CMAKE_INSTALL_PREFIX@/lib64 @CMAKE_INSTALL_PREFIX@/lib NO_DEFAULT_PATH) -find_library(GMP_LIBRARIES NAMES libgmp.a gmp.lib gmp libgmp-10 mpir - HINTS ENV GMP_LIB_DIR - ENV GMP_DIR - PATH_SUFFIXES lib - DOC "Path to the GMP library" -) + +#Ubuntu build requires rt library to be specified explicitly +if(UNIX AND NOT APPLE) + find_library(LIBRT rt) +endif() set(EOSIO_WASM_RUNTIMES @EOSIO_WASM_RUNTIMES@) if("eos-vm-oc" IN_LIST EOSIO_WASM_RUNTIMES) @@ -91,9 +94,7 @@ macro(add_eosio_test_executable test_name) ${libfc} ${libwast} ${libwasm} - ${libwabt} ${libruntime} - ${libplatform} ${libir} ${libsoftfloat} ${liboscrypto} @@ -102,7 +103,7 @@ macro(add_eosio_test_executable test_name) ${libchainbase} ${libbuiltins} ${libsecp256k1} - ${GMP_LIBRARIES} + @GMP_LIBRARIES@ ${Boost_FILESYSTEM_LIBRARY} ${Boost_SYSTEM_LIBRARY} @@ -116,7 +117,14 @@ macro(add_eosio_test_executable test_name) ${PLATFORM_SPECIFIC_LIBS} ${WRAP_MAIN} + Threads::Threads + ${librocksdb} ) + + #adds -ltr. Ubuntu eosio.contracts build breaks without this + if(UNIX AND NOT APPLE) + target_link_libraries(${test_name} ${LIBRT}) + endif() target_include_directories( ${test_name} PUBLIC ${Boost_INCLUDE_DIRS} diff --git a/CMakeModules/EosioTesterBuild.cmake.in b/CMakeModules/EosioTesterBuild.cmake.in index 62704dcf78d..5646bcca276 100644 --- a/CMakeModules/EosioTesterBuild.cmake.in +++ b/CMakeModules/EosioTesterBuild.cmake.in @@ -28,6 +28,10 @@ set( CMAKE_CXX_STANDARD 17 ) set( CMAKE_CXX_EXTENSIONS ON ) set( CXX_STANDARD_REQUIRED ON ) +set(CMAKE_THREAD_PREFER_PTHREAD TRUE) +set(THREADS_PREFER_PTHREAD_FLAG TRUE) +find_package(Threads) + if ( APPLE ) set( CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_CXX_FLAGS} -Wall -Wno-deprecated-declarations" ) else ( APPLE ) @@ -37,7 +41,7 @@ endif ( APPLE ) ### Remove after Boost 1.70 CMake fixes are in place set( Boost_NO_BOOST_CMAKE ON CACHE STRING "ON or OFF" ) set( Boost_USE_STATIC_LIBS ON CACHE STRING "ON or OFF" ) -find_package(Boost 1.67 REQUIRED COMPONENTS +find_package(Boost @Boost_MAJOR_VERSION@.@Boost_MINOR_VERSION@ EXACT REQUIRED COMPONENTS date_time filesystem system @@ -45,6 +49,7 @@ find_package(Boost 1.67 REQUIRED COMPONENTS iostreams unit_test_framework) +find_library(librocksdb rocksdb @CMAKE_BINARY_DIR@/libraries/rocksdb) find_library(libtester eosio_testing @CMAKE_BINARY_DIR@/libraries/testing NO_DEFAULT_PATH) find_library(libchain eosio_chain @CMAKE_BINARY_DIR@/libraries/chain NO_DEFAULT_PATH) if ( "${CMAKE_BUILD_TYPE}" STREQUAL "Debug" ) @@ -59,8 +64,6 @@ endif() find_library(libwasm WASM @CMAKE_BINARY_DIR@/libraries/wasm-jit/Source/WASM NO_DEFAULT_PATH) find_library(libwast WAST @CMAKE_BINARY_DIR@/libraries/wasm-jit/Source/WAST NO_DEFAULT_PATH) find_library(libir IR @CMAKE_BINARY_DIR@/libraries/wasm-jit/Source/IR NO_DEFAULT_PATH) -find_library(libwabt wabt @CMAKE_BINARY_DIR@/libraries/wabt NO_DEFAULT_PATH) -find_library(libplatform Platform @CMAKE_BINARY_DIR@/libraries/wasm-jit/Source/Platform NO_DEFAULT_PATH) find_library(liblogging Logging @CMAKE_BINARY_DIR@/libraries/wasm-jit/Source/Logging NO_DEFAULT_PATH) find_library(libruntime Runtime @CMAKE_BINARY_DIR@/libraries/wasm-jit/Source/Runtime NO_DEFAULT_PATH) find_library(libsoftfloat softfloat @CMAKE_BINARY_DIR@/libraries/softfloat NO_DEFAULT_PATH) @@ -70,12 +73,11 @@ get_filename_component(ssldir @OPENSSL_SSL_LIBRARY@ DIRECTORY) find_library(libosssl ssl "${ssldir}" NO_DEFAULT_PATH) find_library(libchainbase chainbase @CMAKE_BINARY_DIR@/libraries/chainbase NO_DEFAULT_PATH) find_library(libbuiltins builtins @CMAKE_BINARY_DIR@/libraries/builtins NO_DEFAULT_PATH) -find_library(GMP_LIBRARIES NAMES libgmp.a gmp.lib gmp libgmp-10 mpir - HINTS ENV GMP_LIB_DIR - ENV GMP_DIR - PATH_SUFFIXES lib - DOC "Path to the GMP library" -) + +#Ubuntu build requires rt library to be specified explicitly +if(UNIX AND NOT APPLE) + find_library(LIBRT rt) +endif() set(EOSIO_WASM_RUNTIMES @EOSIO_WASM_RUNTIMES@) if("eos-vm-oc" IN_LIST EOSIO_WASM_RUNTIMES) @@ -90,9 +92,7 @@ macro(add_eosio_test_executable test_name) ${libfc} ${libwast} ${libwasm} - ${libwabt} ${libruntime} - ${libplatform} ${libir} ${libsoftfloat} ${liboscrypto} @@ -101,7 +101,7 @@ macro(add_eosio_test_executable test_name) ${libchainbase} ${libbuiltins} ${libsecp256k1} - ${GMP_LIBRARIES} + @GMP_LIBRARIES@ ${Boost_FILESYSTEM_LIBRARY} ${Boost_SYSTEM_LIBRARY} @@ -115,7 +115,14 @@ macro(add_eosio_test_executable test_name) ${PLATFORM_SPECIFIC_LIBS} ${WRAP_MAIN} + Threads::Threads + ${librocksdb} ) + + #adds -ltr. Ubuntu eosio.contracts build breaks without this + if(UNIX AND NOT APPLE) + target_link_libraries(${test_name} ${LIBRT}) + endif() target_include_directories( ${test_name} PUBLIC ${Boost_INCLUDE_DIRS} diff --git a/CMakeModules/MASSigning.cmake b/CMakeModules/MASSigning.cmake index 41c13588f28..2e63fefbd6d 100644 --- a/CMakeModules/MASSigning.cmake +++ b/CMakeModules/MASSigning.cmake @@ -1,18 +1,36 @@ -macro(mas_sign target) +if(APPLE AND MAS_CERT_FINGERPRINT AND MAS_BASE_APPID AND MAS_PROVISIONING_PROFILE) + set(MAS_ENABLED ON) + if(MAS_KEYCHAIN_GROUP) + set(MAS_LEGACY ON) + endif() +endif() + +macro(mas_sign target) #optional argument that forces keygroup when MAS_KEYCHAIN_GROUP not set #example values: # MAS_CERT_FINGERPRINT=C5139C2C4D7FA071EFBFD86CE44B652631C9376A # MAS_BASE_APPID=5A4683969Z.com.example. <) + endif() add_custom_command(TARGET ${target} POST_BUILD - COMMAND ${CMAKE_SOURCE_DIR}/tools/mas_sign.sh ${MAS_CERT_FINGERPRINT} ${MAS_BASE_APPID}$ ${MAS_PROVISIONING_PROFILE} ${MAS_KEYCHAIN_GROUP} $ + COMMAND ${CMAKE_SOURCE_DIR}/tools/mas_sign.sh ${MAS_CERT_FINGERPRINT} ${MAS_BASE_APPID}$ ${MAS_PROVISIONING_PROFILE} ${COMPUTED_KEYCHAIN} $ WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} VERBATIM ) diff --git a/LICENSE b/LICENSE index 463694e9590..df058142c33 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2017-2020 block.one and its contributors. All rights reserved. +Copyright (c) 2017-2021 block.one and its contributors. All rights reserved. The MIT License diff --git a/README.md b/README.md index 3a1399cdc4e..af9863637b3 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,13 @@ EOSIO currently supports the following operating systems: 1. Amazon Linux 2 2. CentOS 7 +2. CentOS 7.x +2. CentOS 8 3. Ubuntu 16.04 4. Ubuntu 18.04 +4. Ubuntu 20.04 5. MacOS 10.14 (Mojave) +6. MacOS 10.15 (Catalina) --- @@ -72,15 +76,20 @@ brew remove eosio ### Ubuntu Linux: +#### Ubuntu 20.04 Package Install +```sh +wget https://github.com/eosio/eos/releases/download/v2.1.0/eosio_2.1.0-1-ubuntu-20.04_amd64.deb +sudo apt install ./eosio_2.1.0-1-ubuntu-20.04_amd64.deb +``` #### Ubuntu 18.04 Package Install ```sh -wget https://github.com/eosio/eos/releases/download/v2.0.12/eosio_2.0.12-1-ubuntu-18.04_amd64.deb -sudo apt install ./eosio_2.0.12-1-ubuntu-18.04_amd64.deb +wget https://github.com/eosio/eos/releases/download/v2.1.0/eosio_2.1.0-1-ubuntu-18.04_amd64.deb +sudo apt install ./eosio_2.1.0-1-ubuntu-18.04_amd64.deb ``` #### Ubuntu 16.04 Package Install ```sh -wget https://github.com/eosio/eos/releases/download/v2.0.12/eosio_2.0.12-1-ubuntu-16.04_amd64.deb -sudo apt install ./eosio_2.0.12-1-ubuntu-16.04_amd64.deb +wget https://github.com/eosio/eos/releases/download/v2.1.0/eosio_2.1.0-1-ubuntu-16.04_amd64.deb +sudo apt install ./eosio_2.1.0-1-ubuntu-16.04_amd64.deb ``` #### Ubuntu Package Uninstall ```sh @@ -89,11 +98,17 @@ sudo apt remove eosio ### RPM-based (CentOS, Amazon Linux, etc.): -#### RPM Package Install +#### RPM Package Install CentOS 7 ```sh -wget https://github.com/eosio/eos/releases/download/v2.0.12/eosio-2.0.12-1.el7.x86_64.rpm -sudo yum install ./eosio-2.0.12-1.el7.x86_64.rpm +wget https://github.com/eosio/eos/releases/download/v2.1.0/eosio-2.1.0-1.el7.x86_64.rpm +sudo yum install ./eosio-2.1.0-1.el7.x86_64.rpm ``` +#### RPM Package Install CentOS 8 +```sh +wget https://github.com/eosio/eos/releases/download/v2.1.0/eosio-2.1.0-1.el8.x86_64.rpm +sudo yum install ./eosio-2.1.0-1.el8.x86_64.rpm +``` + #### RPM Package Uninstall ```sh sudo yum remove eosio @@ -124,9 +139,8 @@ To uninstall the EOSIO built/installed binaries and dependencies, run: 1. [White Paper](https://github.com/EOSIO/Documentation/blob/master/TechnicalWhitePaper.md) 1. [Roadmap](https://github.com/EOSIO/Documentation/blob/master/Roadmap.md) - ## Getting Started -Instructions detailing the process of getting the software, building it, running a simple test network that produces blocks, account creation and uploading a sample contract to the blockchain can be found in the [Getting Started](https://developers.eos.io/welcome/latest/getting-started-guide) walkthrough. +Instructions detailing the process of getting the software, building it, running a simple test network that produces blocks, account creation and uploading a sample contract to the blockchain can be found in the [Getting Started](https://developers.eos.io/welcome/v2.1/getting-started-guide) walkthrough. ## Contributing diff --git a/RocksDB.License.Apache b/RocksDB.License.Apache new file mode 100644 index 00000000000..7a4a3ea2424 --- /dev/null +++ b/RocksDB.License.Apache @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/contracts/CMakeLists.txt b/contracts/CMakeLists.txt new file mode 100644 index 00000000000..49a54136a86 --- /dev/null +++ b/contracts/CMakeLists.txt @@ -0,0 +1,27 @@ +### Build contracts with cdt if EOSIO_COMPILE_TEST_CONTRACTS is on +include(ExternalProject) + +if( EOSIO_COMPILE_TEST_CONTRACTS ) + set(EOSIO_WASM_OLD_BEHAVIOR "Off") + find_package(eosio.cdt REQUIRED) + + set(CMAKE_ARGS_VAL -DCMAKE_TOOLCHAIN_FILE=${EOSIO_CDT_ROOT}/lib/cmake/eosio.cdt/EosioWasmToolchain.cmake -DEOSIO_COMPILE_TEST_CONTRACTS=${EOSIO_COMPILE_TEST_CONTRACTS} ) + if( USE_EOSIO_CDT_1_7_X) + list(APPEND CMAKE_ARGS_VAL -DUSE_EOSIO_CDT_1_7_X=${USE_EOSIO_CDT_1_7_X}) + endif() + message( STATUS "Building contracts in directory `eos/contracts/`" ) + ExternalProject_Add( + bios_boot_contracts_project + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/contracts + BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/contracts + CMAKE_ARGS ${CMAKE_ARGS_VAL} + UPDATE_COMMAND "" + PATCH_COMMAND "" + TEST_COMMAND "" + INSTALL_COMMAND "" + BUILD_ALWAYS 1 + ) +else() + message( STATUS "Not building contracts in directory `eos/contracts/`" ) + add_subdirectory(contracts) +endif() diff --git a/contracts/contracts/CMakeLists.txt b/contracts/contracts/CMakeLists.txt new file mode 100644 index 00000000000..4a5712b10b5 --- /dev/null +++ b/contracts/contracts/CMakeLists.txt @@ -0,0 +1,7 @@ +if( EOSIO_COMPILE_TEST_CONTRACTS ) + set(EOSIO_WASM_OLD_BEHAVIOR "Off") + find_package( eosio.cdt REQUIRED ) +endif() + +add_subdirectory(eosio.bios) +add_subdirectory(eosio.boot) diff --git a/contracts/contracts/eosio.bios/CMakeLists.txt b/contracts/contracts/eosio.bios/CMakeLists.txt new file mode 100644 index 00000000000..94cc2a8463a --- /dev/null +++ b/contracts/contracts/eosio.bios/CMakeLists.txt @@ -0,0 +1,18 @@ +if (EOSIO_COMPILE_TEST_CONTRACTS) + add_contract(eosio.bios eosio.bios ${CMAKE_CURRENT_SOURCE_DIR}/src/eosio.bios.cpp) + + target_include_directories(eosio.bios + PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/include) + + set_target_properties(eosio.bios + PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") + + configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/ricardian/eosio.bios.contracts.md.in ${CMAKE_CURRENT_BINARY_DIR}/ricardian/eosio.bios.contracts.md @ONLY ) + + target_compile_options( eosio.bios PUBLIC -R${CMAKE_CURRENT_SOURCE_DIR}/ricardian -R${CMAKE_CURRENT_BINARY_DIR}/ricardian ) +else() + configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/bin/eosio.bios.abi ${CMAKE_CURRENT_BINARY_DIR}/ COPYONLY ) + configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/bin/eosio.bios.wasm ${CMAKE_CURRENT_BINARY_DIR}/ COPYONLY ) +endif() diff --git a/contracts/contracts/eosio.bios/bin/eosio.bios.abi b/contracts/contracts/eosio.bios/bin/eosio.bios.abi new file mode 100644 index 00000000000..8b73b2e273c --- /dev/null +++ b/contracts/contracts/eosio.bios/bin/eosio.bios.abi @@ -0,0 +1,602 @@ +{ + "____comment": "This file was generated with eosio-abigen. DO NOT EDIT ", + "version": "eosio::abi/1.2", + "types": [ + { + "new_type_name": "block_signing_authority", + "type": "variant_block_signing_authority_v0" + } + ], + "structs": [ + { + "name": "abi_hash", + "base": "", + "fields": [ + { + "name": "owner", + "type": "name" + }, + { + "name": "hash", + "type": "checksum256" + } + ] + }, + { + "name": "activate", + "base": "", + "fields": [ + { + "name": "feature_digest", + "type": "checksum256" + } + ] + }, + { + "name": "authority", + "base": "", + "fields": [ + { + "name": "threshold", + "type": "uint32" + }, + { + "name": "keys", + "type": "key_weight[]" + }, + { + "name": "accounts", + "type": "permission_level_weight[]" + }, + { + "name": "waits", + "type": "wait_weight[]" + } + ] + }, + { + "name": "block_signing_authority_v0", + "base": "", + "fields": [ + { + "name": "threshold", + "type": "uint32" + }, + { + "name": "keys", + "type": "key_weight[]" + } + ] + }, + { + "name": "blockchain_parameters", + "base": "", + "fields": [ + { + "name": "max_block_net_usage", + "type": "uint64" + }, + { + "name": "target_block_net_usage_pct", + "type": "uint32" + }, + { + "name": "max_transaction_net_usage", + "type": "uint32" + }, + { + "name": "base_per_transaction_net_usage", + "type": "uint32" + }, + { + "name": "net_usage_leeway", + "type": "uint32" + }, + { + "name": "context_free_discount_net_usage_num", + "type": "uint32" + }, + { + "name": "context_free_discount_net_usage_den", + "type": "uint32" + }, + { + "name": "max_block_cpu_usage", + "type": "uint32" + }, + { + "name": "target_block_cpu_usage_pct", + "type": "uint32" + }, + { + "name": "max_transaction_cpu_usage", + "type": "uint32" + }, + { + "name": "min_transaction_cpu_usage", + "type": "uint32" + }, + { + "name": "max_transaction_lifetime", + "type": "uint32" + }, + { + "name": "deferred_trx_expiration_window", + "type": "uint32" + }, + { + "name": "max_transaction_delay", + "type": "uint32" + }, + { + "name": "max_inline_action_size", + "type": "uint32" + }, + { + "name": "max_inline_action_depth", + "type": "uint16" + }, + { + "name": "max_authority_depth", + "type": "uint16" + } + ] + }, + { + "name": "canceldelay", + "base": "", + "fields": [ + { + "name": "canceling_auth", + "type": "permission_level" + }, + { + "name": "trx_id", + "type": "checksum256" + } + ] + }, + { + "name": "deleteauth", + "base": "", + "fields": [ + { + "name": "account", + "type": "name" + }, + { + "name": "permission", + "type": "name" + } + ] + }, + { + "name": "key_weight", + "base": "", + "fields": [ + { + "name": "key", + "type": "public_key" + }, + { + "name": "weight", + "type": "uint16" + } + ] + }, + { + "name": "kv_parameters", + "base": "", + "fields": [ + { + "name": "max_key_size", + "type": "uint32" + }, + { + "name": "max_value_size", + "type": "uint32" + }, + { + "name": "max_iterators", + "type": "uint32" + } + ] + }, + { + "name": "linkauth", + "base": "", + "fields": [ + { + "name": "account", + "type": "name" + }, + { + "name": "code", + "type": "name" + }, + { + "name": "type", + "type": "name" + }, + { + "name": "requirement", + "type": "name" + } + ] + }, + { + "name": "newaccount", + "base": "", + "fields": [ + { + "name": "creator", + "type": "name" + }, + { + "name": "name", + "type": "name" + }, + { + "name": "owner", + "type": "authority" + }, + { + "name": "active", + "type": "authority" + } + ] + }, + { + "name": "onerror", + "base": "", + "fields": [ + { + "name": "sender_id", + "type": "uint128" + }, + { + "name": "sent_trx", + "type": "bytes" + } + ] + }, + { + "name": "permission_level", + "base": "", + "fields": [ + { + "name": "actor", + "type": "name" + }, + { + "name": "permission", + "type": "name" + } + ] + }, + { + "name": "permission_level_weight", + "base": "", + "fields": [ + { + "name": "permission", + "type": "permission_level" + }, + { + "name": "weight", + "type": "uint16" + } + ] + }, + { + "name": "producer_authority", + "base": "", + "fields": [ + { + "name": "producer_name", + "type": "name" + }, + { + "name": "authority", + "type": "block_signing_authority" + } + ] + }, + { + "name": "reqactivated", + "base": "", + "fields": [ + { + "name": "feature_digest", + "type": "checksum256" + } + ] + }, + { + "name": "reqauth", + "base": "", + "fields": [ + { + "name": "from", + "type": "name" + } + ] + }, + { + "name": "setabi", + "base": "", + "fields": [ + { + "name": "account", + "type": "name" + }, + { + "name": "abi", + "type": "bytes" + } + ] + }, + { + "name": "setalimits", + "base": "", + "fields": [ + { + "name": "account", + "type": "name" + }, + { + "name": "ram_bytes", + "type": "int64" + }, + { + "name": "net_weight", + "type": "int64" + }, + { + "name": "cpu_weight", + "type": "int64" + } + ] + }, + { + "name": "setcode", + "base": "", + "fields": [ + { + "name": "account", + "type": "name" + }, + { + "name": "vmtype", + "type": "uint8" + }, + { + "name": "vmversion", + "type": "uint8" + }, + { + "name": "code", + "type": "bytes" + } + ] + }, + { + "name": "setkvparams", + "base": "", + "fields": [ + { + "name": "params", + "type": "kv_parameters" + } + ] + }, + { + "name": "setparams", + "base": "", + "fields": [ + { + "name": "params", + "type": "blockchain_parameters" + } + ] + }, + { + "name": "setpparams", + "base": "", + "fields": [ + { + "name": "bytes", + "type": "bytes" + } + ] + }, + { + "name": "setpriv", + "base": "", + "fields": [ + { + "name": "account", + "type": "name" + }, + { + "name": "is_priv", + "type": "uint8" + } + ] + }, + { + "name": "setprods", + "base": "", + "fields": [ + { + "name": "schedule", + "type": "producer_authority[]" + } + ] + }, + { + "name": "unlinkauth", + "base": "", + "fields": [ + { + "name": "account", + "type": "name" + }, + { + "name": "code", + "type": "name" + }, + { + "name": "type", + "type": "name" + } + ] + }, + { + "name": "updateauth", + "base": "", + "fields": [ + { + "name": "account", + "type": "name" + }, + { + "name": "permission", + "type": "name" + }, + { + "name": "parent", + "type": "name" + }, + { + "name": "auth", + "type": "authority" + } + ] + }, + { + "name": "wait_weight", + "base": "", + "fields": [ + { + "name": "wait_sec", + "type": "uint32" + }, + { + "name": "weight", + "type": "uint16" + } + ] + } + ], + "actions": [ + { + "name": "activate", + "type": "activate", + "ricardian_contract": "" + }, + { + "name": "canceldelay", + "type": "canceldelay", + "ricardian_contract": "" + }, + { + "name": "deleteauth", + "type": "deleteauth", + "ricardian_contract": "" + }, + { + "name": "linkauth", + "type": "linkauth", + "ricardian_contract": "" + }, + { + "name": "newaccount", + "type": "newaccount", + "ricardian_contract": "" + }, + { + "name": "onerror", + "type": "onerror", + "ricardian_contract": "" + }, + { + "name": "reqactivated", + "type": "reqactivated", + "ricardian_contract": "" + }, + { + "name": "reqauth", + "type": "reqauth", + "ricardian_contract": "" + }, + { + "name": "setabi", + "type": "setabi", + "ricardian_contract": "" + }, + { + "name": "setalimits", + "type": "setalimits", + "ricardian_contract": "" + }, + { + "name": "setcode", + "type": "setcode", + "ricardian_contract": "" + }, + { + "name": "setkvparams", + "type": "setkvparams", + "ricardian_contract": "" + }, + { + "name": "setparams", + "type": "setparams", + "ricardian_contract": "" + }, + { + "name": "setpparams", + "type": "setpparams", + "ricardian_contract": "" + }, + { + "name": "setpriv", + "type": "setpriv", + "ricardian_contract": "" + }, + { + "name": "setprods", + "type": "setprods", + "ricardian_contract": "" + }, + { + "name": "unlinkauth", + "type": "unlinkauth", + "ricardian_contract": "" + }, + { + "name": "updateauth", + "type": "updateauth", + "ricardian_contract": "" + } + ], + "tables": [ + { + "name": "abihash", + "type": "abi_hash", + "index_type": "i64", + "key_names": [], + "key_types": [] + } + ], + "kv_tables": {}, + "ricardian_clauses": [], + "variants": [ + { + "name": "variant_block_signing_authority_v0", + "types": ["block_signing_authority_v0"] + } + ], + "action_results": [] +} \ No newline at end of file diff --git a/contracts/contracts/eosio.bios/bin/eosio.bios.wasm b/contracts/contracts/eosio.bios/bin/eosio.bios.wasm new file mode 100755 index 00000000000..758bef069b8 Binary files /dev/null and b/contracts/contracts/eosio.bios/bin/eosio.bios.wasm differ diff --git a/contracts/contracts/eosio.bios/include/eosio.bios/eosio.bios.hpp b/contracts/contracts/eosio.bios/include/eosio.bios/eosio.bios.hpp new file mode 100644 index 00000000000..63b37d3e50a --- /dev/null +++ b/contracts/contracts/eosio.bios/include/eosio.bios/eosio.bios.hpp @@ -0,0 +1,300 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace eosiobios { + + using eosio::action_wrapper; + using eosio::check; + using eosio::checksum256; + using eosio::ignore; + using eosio::name; + using eosio::permission_level; + using eosio::public_key; + + struct permission_level_weight { + permission_level permission; + uint16_t weight; + + // explicit serialization macro is not necessary, used here only to improve compilation time + EOSLIB_SERIALIZE( permission_level_weight, (permission)(weight) ) + }; + + struct key_weight { + eosio::public_key key; + uint16_t weight; + + // explicit serialization macro is not necessary, used here only to improve compilation time + EOSLIB_SERIALIZE( key_weight, (key)(weight) ) + }; + + struct wait_weight { + uint32_t wait_sec; + uint16_t weight; + + // explicit serialization macro is not necessary, used here only to improve compilation time + EOSLIB_SERIALIZE( wait_weight, (wait_sec)(weight) ) + }; + + struct authority { + uint32_t threshold = 0; + std::vector keys; + std::vector accounts; + std::vector waits; + + // explicit serialization macro is not necessary, used here only to improve compilation time + EOSLIB_SERIALIZE( authority, (threshold)(keys)(accounts)(waits) ) + }; + + struct block_header { + uint32_t timestamp; + name producer; + uint16_t confirmed = 0; + checksum256 previous; + checksum256 transaction_mroot; + checksum256 action_mroot; + uint32_t schedule_version = 0; + std::optional new_producers; + + // explicit serialization macro is not necessary, used here only to improve compilation time + EOSLIB_SERIALIZE(block_header, (timestamp)(producer)(confirmed)(previous)(transaction_mroot)(action_mroot) + (schedule_version)(new_producers)) + }; + + /** + * The `eosio.bios` is the first sample of system contract provided by `block.one` through the EOSIO platform. It is a minimalist system contract because it only supplies the actions that are absolutely critical to bootstrap a chain and nothing more. This allows for a chain agnostic approach to bootstrapping a chain. + * + * Just like in the `eosio.system` sample contract implementation, there are a few actions which are not implemented at the contract level (`newaccount`, `updateauth`, `deleteauth`, `linkauth`, `unlinkauth`, `canceldelay`, `onerror`, `setabi`, `setcode`), they are just declared in the contract so they will show in the contract's ABI and users will be able to push those actions to the chain via the account holding the `eosio.system` contract, but the implementation is at the EOSIO core level. They are referred to as EOSIO native actions. + */ + class [[eosio::contract("eosio.bios")]] bios : public eosio::contract { + public: + using contract::contract; + /** + * New account action, called after a new account is created. This code enforces resource-limits rules + * for new accounts as well as new account naming conventions. + * + * 1. accounts cannot contain '.' symbols which forces all acccounts to be 12 + * characters long without '.' until a future account auction process is implemented + * which prevents name squatting. + * + * 2. new accounts must stake a minimal number of tokens (as set in system parameters) + * therefore, this method will execute an inline buyram from receiver for newacnt in + * an amount equal to the current new account creation fee. + */ + [[eosio::action]] + void newaccount( name creator, + name name, + ignore owner, + ignore active){} + /** + * Update authorization action updates pemission for an account. + * + * @param account - the account for which the permission is updated, + * @param pemission - the permission name which is updated, + * @param parem - the parent of the permission which is updated, + * @param aut - the json describing the permission authorization. + */ + [[eosio::action]] + void updateauth( ignore account, + ignore permission, + ignore parent, + ignore auth ) {} + + /** + * Delete authorization action deletes the authorization for an account's permission. + * + * @param account - the account for which the permission authorization is deleted, + * @param permission - the permission name been deleted. + */ + [[eosio::action]] + void deleteauth( ignore account, + ignore permission ) {} + + /** + * Link authorization action assigns a specific action from a contract to a permission you have created. Five system + * actions can not be linked `updateauth`, `deleteauth`, `linkauth`, `unlinkauth`, and `canceldelay`. + * This is useful because when doing authorization checks, the EOSIO based blockchain starts with the + * action needed to be authorized (and the contract belonging to), and looks up which permission + * is needed to pass authorization validation. If a link is set, that permission is used for authoraization + * validation otherwise then active is the default, with the exception of `eosio.any`. + * `eosio.any` is an implicit permission which exists on every account; you can link actions to `eosio.any` + * and that will make it so linked actions are accessible to any permissions defined for the account. + * + * @param account - the permission's owner to be linked and the payer of the RAM needed to store this link, + * @param code - the owner of the action to be linked, + * @param type - the action to be linked, + * @param requirement - the permission to be linked. + */ + [[eosio::action]] + void linkauth( ignore account, + ignore code, + ignore type, + ignore requirement ) {} + + /** + * Unlink authorization action it's doing the reverse of linkauth action, by unlinking the given action. + * + * @param account - the owner of the permission to be unlinked and the receiver of the freed RAM, + * @param code - the owner of the action to be unlinked, + * @param type - the action to be unlinked. + */ + [[eosio::action]] + void unlinkauth( ignore account, + ignore code, + ignore type ) {} + + /** + * Cancel delay action cancels a deferred transaction. + * + * @param canceling_auth - the permission that authorizes this action, + * @param trx_id - the deferred transaction id to be cancelled. + */ + [[eosio::action]] + void canceldelay( ignore canceling_auth, ignore trx_id ) {} + + /** + * Set code action sets the contract code for an account. + * + * @param account - the account for which to set the contract code. + * @param vmtype - reserved, set it to zero. + * @param vmversion - reserved, set it to zero. + * @param code - the code content to be set, in the form of a blob binary.. + */ + [[eosio::action]] + void setcode( name account, uint8_t vmtype, uint8_t vmversion, const std::vector& code ) {} + + /** + * Set abi action sets the abi for contract identified by `account` name. Creates an entry in the abi_hash_table + * index, with `account` name as key, if it is not already present and sets its value with the abi hash. + * Otherwise it is updating the current abi hash value for the existing `account` key. + * + * @param account - the name of the account to set the abi for + * @param abi - the abi hash represented as a vector of characters + */ + [[eosio::action]] + void setabi( name account, const std::vector& abi ); + + /** + * On error action, notification of this action is delivered to the sender of a deferred transaction + * when an objective error occurs while executing the deferred transaction. + * This action is not meant to be called directly. + * + * @param sender_id - the id for the deferred transaction chosen by the sender, + * @param sent_trx - the deferred transaction that failed. + */ + [[eosio::action]] + void onerror( ignore sender_id, ignore> sent_trx ); + + /** + * Set privilege action allows to set privilege status for an account (turn it on/off). + * @param account - the account to set the privileged status for. + * @param is_priv - 0 for false, > 0 for true. + */ + [[eosio::action]] + void setpriv( name account, uint8_t is_priv ); + + /** + * Sets the resource limits of an account + * + * @param account - name of the account whose resource limit to be set + * @param ram_bytes - ram limit in absolute bytes + * @param net_weight - fractionally proportionate net limit of available resources based on (weight / total_weight_of_all_accounts) + * @param cpu_weight - fractionally proportionate cpu limit of available resources based on (weight / total_weight_of_all_accounts) + */ + [[eosio::action]] + void setalimits( name account, int64_t ram_bytes, int64_t net_weight, int64_t cpu_weight ); + + /** + * Set producers action, sets a new list of active producers, by proposing a schedule change, once the block that + * contains the proposal becomes irreversible, the schedule is promoted to "pending" + * automatically. Once the block that promotes the schedule is irreversible, the schedule will + * become "active". + * + * @param schedule - New list of active producers to set + */ + [[eosio::action]] + void setprods( const std::vector& schedule ); + + /** + * Set params action, sets the blockchain parameters. By tuning these parameters, various degrees of customization can be achieved. + * + * @param params - New blockchain parameters to set + */ + [[eosio::action]] + void setparams( const eosio::blockchain_parameters& params ); + + /** + * Set params action, sets the blockchain parameters. By tuning these parameters, various degrees of customization can be achieved. + * + * @param params - New blockchain parameters to set + */ + [[eosio::action]] + void setpparams( const std::vector& bytes); + + /** + * Set KV params action, sets the KV parameters. By tuning these parameters, various degrees of customization can be achieved. + * + * @param params - New KV parameters to set + */ + [[eosio::action]] + void setkvparams( const eosio::kv_parameters& params ); + + /** + * Require authorization action, checks if the account name `from` passed in as param has authorization to access + * current action, that is, if it is listed in the action’s allowed permissions vector. + * + * @param from - the account name to authorize + */ + [[eosio::action]] + void reqauth( name from ); + + /** + * Activate action, activates a protocol feature + * + * @param feature_digest - hash of the protocol feature to activate. + */ + [[eosio::action]] + void activate( const eosio::checksum256& feature_digest ); + + /** + * Require activated action, asserts that a protocol feature has been activated + * + * @param feature_digest - hash of the protocol feature to check for activation. + */ + [[eosio::action]] + void reqactivated( const eosio::checksum256& feature_digest ); + + struct [[eosio::table]] abi_hash { + name owner; + checksum256 hash; + uint64_t primary_key()const { return owner.value; } + + EOSLIB_SERIALIZE( abi_hash, (owner)(hash) ) + }; + + typedef eosio::multi_index< "abihash"_n, abi_hash > abi_hash_table; + + using newaccount_action = action_wrapper<"newaccount"_n, &bios::newaccount>; + using updateauth_action = action_wrapper<"updateauth"_n, &bios::updateauth>; + using deleteauth_action = action_wrapper<"deleteauth"_n, &bios::deleteauth>; + using linkauth_action = action_wrapper<"linkauth"_n, &bios::linkauth>; + using unlinkauth_action = action_wrapper<"unlinkauth"_n, &bios::unlinkauth>; + using canceldelay_action = action_wrapper<"canceldelay"_n, &bios::canceldelay>; + using setcode_action = action_wrapper<"setcode"_n, &bios::setcode>; + using setabi_action = action_wrapper<"setabi"_n, &bios::setabi>; + using setpriv_action = action_wrapper<"setpriv"_n, &bios::setpriv>; + using setalimits_action = action_wrapper<"setalimits"_n, &bios::setalimits>; + using setprods_action = action_wrapper<"setprods"_n, &bios::setprods>; + using setparams_action = action_wrapper<"setparams"_n, &bios::setparams>; + using setpparams_action = action_wrapper<"setpparams"_n, &bios::setpparams>; + using setkvparams_action = action_wrapper<"setkvparams"_n, &bios::setkvparams>; + using reqauth_action = action_wrapper<"reqauth"_n, &bios::reqauth>; + using activate_action = action_wrapper<"activate"_n, &bios::activate>; + using reqactivated_action = action_wrapper<"reqactivated"_n, &bios::reqactivated>; + }; +} diff --git a/contracts/contracts/eosio.bios/ricardian/eosio.bios.contracts.md.in b/contracts/contracts/eosio.bios/ricardian/eosio.bios.contracts.md.in new file mode 100644 index 00000000000..20d18f8d4ff --- /dev/null +++ b/contracts/contracts/eosio.bios/ricardian/eosio.bios.contracts.md.in @@ -0,0 +1,201 @@ +

activate

+ +--- +spec_version: "0.2.0" +title: Activate Protocol Feature +summary: 'Activate protocol feature {{nowrap feature_digest}}' +icon: @ICON_BASE_URL@/@ADMIN_ICON_URI@ +--- + +{{$action.account}} activates the protocol feature with a digest of {{feature_digest}}. + +

canceldelay

+ +--- +spec_version: "0.2.0" +title: Cancel Delayed Transaction +summary: '{{nowrap canceling_auth.actor}} cancels a delayed transaction' +icon: @ICON_BASE_URL@/@ACCOUNT_ICON_URI@ +--- + +{{canceling_auth.actor}} cancels the delayed transaction with id {{trx_id}}. + +

deleteauth

+ +--- +spec_version: "0.2.0" +title: Delete Account Permission +summary: 'Delete the {{nowrap permission}} permission of {{nowrap account}}' +icon: @ICON_BASE_URL@/@ACCOUNT_ICON_URI@ +--- + +Delete the {{permission}} permission of {{account}}. + +

linkauth

+ +--- +spec_version: "0.2.0" +title: Link Action to Permission +summary: '{{nowrap account}} sets the minimum required permission for the {{#if type}}{{nowrap type}} action of the{{/if}} {{nowrap code}} contract to {{nowrap requirement}}' +icon: @ICON_BASE_URL@/@ACCOUNT_ICON_URI@ +--- + +{{account}} sets the minimum required permission for the {{#if type}}{{type}} action of the{{/if}} {{code}} contract to {{requirement}}. + +{{#if type}}{{else}}Any links explicitly associated to specific actions of {{code}} will take precedence.{{/if}} + +

newaccount

+ +--- +spec_version: "0.2.0" +title: Create New Account +summary: '{{nowrap creator}} creates a new account with the name {{nowrap name}}' +icon: @ICON_BASE_URL@/@ACCOUNT_ICON_URI@ +--- + +{{creator}} creates a new account with the name {{name}} and the following permissions: + +owner permission with authority: +{{to_json owner}} + +active permission with authority: +{{to_json active}} + +

reqactivated

+ +--- +spec_version: "0.2.0" +title: Assert Protocol Feature Activation +summary: 'Assert that protocol feature {{nowrap feature_digest}} has been activated' +icon: @ICON_BASE_URL@/@ADMIN_ICON_URI@ +--- + +Assert that the protocol feature with a digest of {{feature_digest}} has been activated. + +

reqauth

+ +--- +spec_version: "0.2.0" +title: Assert Authorization +summary: 'Assert that authorization by {{nowrap from}} is provided' +icon: @ICON_BASE_URL@/@ACCOUNT_ICON_URI@ +--- + +Assert that authorization by {{from}} is provided. + +

setabi

+ +--- +spec_version: "0.2.0" +title: Deploy Contract ABI +summary: 'Deploy contract ABI on account {{nowrap account}}' +icon: @ICON_BASE_URL@/@ACCOUNT_ICON_URI@ +--- + +Deploy the ABI file associated with the contract on account {{account}}. + +

setalimits

+ +--- +spec_version: "0.2.0" +title: Adjust Resource Limits of Account +summary: 'Adjust resource limits of account {{nowrap account}}' +icon: @ICON_BASE_URL@/@ADMIN_ICON_URI@ +--- + +{{$action.account}} updates {{account}}’s resource limits to have a RAM quota of {{ram_bytes}} bytes, a NET bandwidth quota of {{net_weight}} and a CPU bandwidth quota of {{cpu_weight}}. + +

setcode

+ +--- +spec_version: "0.2.0" +title: Deploy Contract Code +summary: 'Deploy contract code on account {{nowrap account}}' +icon: @ICON_BASE_URL@/@ACCOUNT_ICON_URI@ +--- + +Deploy compiled contract code to the account {{account}}. + +

setparams

+ +--- +spec_version: "0.2.0" +title: Set System Parameters +summary: 'Set system parameters' +icon: @ICON_BASE_URL@/@ADMIN_ICON_URI@ +--- + +{{$action.account}} sets system parameters to: +{{to_json params}} + +

setkvparams

+ +--- +spec_version: "0.3.0" +title: Set KV Parameters +summary: 'Set KV parameters' +icon: @ICON_BASE_URL@/@ADMIN_ICON_URI@ +--- + +{{$action.account}} sets KV parameters to: +{{to_json params}} + +

setpriv

+ +--- +spec_version: "0.2.0" +title: Make an Account Privileged or Unprivileged +summary: '{{#if is_priv}}Make {{nowrap account}} privileged{{else}}Remove privileged status of {{nowrap account}}{{/if}}' +icon: @ICON_BASE_URL@/@ADMIN_ICON_URI@ +--- + +{{#if is_priv}} +{{$action.account}} makes {{account}} privileged. +{{else}} +{{$action.account}} removes privileged status of {{account}}. +{{/if}} + +

setprods

+ +--- +spec_version: "0.2.0" +title: Set Block Producers +summary: 'Set block producer schedule' +icon: @ICON_BASE_URL@/@ADMIN_ICON_URI@ +--- + +{{$action.account}} proposes a block producer schedule of: +{{#each schedule}} + 1. {{this.producer_name}} +{{/each}} + +The block signing authorities of each of the producers in the above schedule are listed below: +{{#each schedule}} +### {{this.producer_name}} +{{to_json this.authority}} +{{/each}} + +

unlinkauth

+ +--- +spec_version: "0.2.0" +title: Unlink Action from Permission +summary: '{{nowrap account}} unsets the minimum required permission for the {{#if type}}{{nowrap type}} action of the{{/if}} {{nowrap code}} contract' +icon: @ICON_BASE_URL@/@ACCOUNT_ICON_URI@ +--- + +{{account}} removes the association between the {{#if type}}{{type}} action of the{{/if}} {{code}} contract and its minimum required permission. + +{{#if type}}{{else}}This will not remove any links explicitly associated to specific actions of {{code}}.{{/if}} + +

updateauth

+ +--- +spec_version: "0.2.0" +title: Modify Account Permission +summary: 'Add or update the {{nowrap permission}} permission of {{nowrap account}}' +icon: @ICON_BASE_URL@/@ACCOUNT_ICON_URI@ +--- + +Modify, and create if necessary, the {{permission}} permission of {{account}} to have a parent permission of {{parent}} and the following authority: +{{to_json auth}} diff --git a/contracts/contracts/eosio.bios/src/eosio.bios.cpp b/contracts/contracts/eosio.bios/src/eosio.bios.cpp new file mode 100644 index 00000000000..a87961d8c89 --- /dev/null +++ b/contracts/contracts/eosio.bios/src/eosio.bios.cpp @@ -0,0 +1,73 @@ +#include + +namespace eosiobios { + +// move this to CDT after this release +extern "C" { + __attribute__((eosio_wasm_import)) + void set_parameters_packed(const char*, std::size_t); +} + +void bios::setabi( name account, const std::vector& abi ) { + abi_hash_table table(get_self(), get_self().value); + auto itr = table.find( account.value ); + if( itr == table.end() ) { + table.emplace( account, [&]( auto& row ) { + row.owner = account; + row.hash = eosio::sha256(const_cast(abi.data()), abi.size()); + }); + } else { + table.modify( itr, eosio::same_payer, [&]( auto& row ) { + row.hash = eosio::sha256(const_cast(abi.data()), abi.size()); + }); + } +} + +void bios::onerror( ignore, ignore> ) { + check( false, "the onerror action cannot be called directly" ); +} + +void bios::setpriv( name account, uint8_t is_priv ) { + require_auth( get_self() ); + set_privileged( account, is_priv ); +} + +void bios::setalimits( name account, int64_t ram_bytes, int64_t net_weight, int64_t cpu_weight ) { + require_auth( get_self() ); + set_resource_limits( account, ram_bytes, net_weight, cpu_weight ); +} + +void bios::setprods( const std::vector& schedule ) { + require_auth( get_self() ); + set_proposed_producers( schedule ); +} + +void bios::setparams( const eosio::blockchain_parameters& params ) { + require_auth( get_self() ); + set_blockchain_parameters( params ); +} + +void bios::setpparams( const std::vector& params ) { + require_auth( get_self() ); + set_parameters_packed( params.data(), params.size() ); +} + +void bios::setkvparams( const eosio::kv_parameters& params ) { + require_auth( get_self() ); + set_kv_parameters( params ); +} + +void bios::reqauth( name from ) { + require_auth( from ); +} + +void bios::activate( const eosio::checksum256& feature_digest ) { + require_auth( get_self() ); + preactivate_feature( feature_digest ); +} + +void bios::reqactivated( const eosio::checksum256& feature_digest ) { + check( is_feature_activated( feature_digest ), "protocol feature is not activated" ); +} + +} diff --git a/contracts/contracts/eosio.boot/CMakeLists.txt b/contracts/contracts/eosio.boot/CMakeLists.txt new file mode 100644 index 00000000000..2b53d1f898f --- /dev/null +++ b/contracts/contracts/eosio.boot/CMakeLists.txt @@ -0,0 +1,19 @@ +if (EOSIO_COMPILE_TEST_CONTRACTS) + add_contract(eosio.boot eosio.boot ${CMAKE_CURRENT_SOURCE_DIR}/src/eosio.boot.cpp) + + target_include_directories(eosio.boot + PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/include) + + set_target_properties(eosio.boot + PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") + + configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/ricardian/eosio.boot.contracts.md.in ${CMAKE_CURRENT_BINARY_DIR}/ricardian/eosio.boot.contracts.md @ONLY ) + + target_compile_options( eosio.boot PUBLIC -R${CMAKE_CURRENT_SOURCE_DIR}/ricardian -R${CMAKE_CURRENT_BINARY_DIR}/ricardian ) +else() + configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/bin/eosio.boot.abi ${CMAKE_CURRENT_BINARY_DIR}/ COPYONLY ) + configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/bin/eosio.boot.wasm ${CMAKE_CURRENT_BINARY_DIR}/ COPYONLY ) +endif() + diff --git a/contracts/contracts/eosio.boot/bin/eosio.boot.abi b/contracts/contracts/eosio.boot/bin/eosio.boot.abi new file mode 100644 index 00000000000..fecbd61e9d6 --- /dev/null +++ b/contracts/contracts/eosio.boot/bin/eosio.boot.abi @@ -0,0 +1,329 @@ +{ + "____comment": "This file was generated with eosio-abigen. DO NOT EDIT ", + "version": "eosio::abi/1.2", + "types": [], + "structs": [ + { + "name": "activate", + "base": "", + "fields": [ + { + "name": "feature_digest", + "type": "checksum256" + } + ] + }, + { + "name": "authority", + "base": "", + "fields": [ + { + "name": "threshold", + "type": "uint32" + }, + { + "name": "keys", + "type": "key_weight[]" + }, + { + "name": "accounts", + "type": "permission_level_weight[]" + }, + { + "name": "waits", + "type": "wait_weight[]" + } + ] + }, + { + "name": "canceldelay", + "base": "", + "fields": [ + { + "name": "canceling_auth", + "type": "permission_level" + }, + { + "name": "trx_id", + "type": "checksum256" + } + ] + }, + { + "name": "deleteauth", + "base": "", + "fields": [ + { + "name": "account", + "type": "name" + }, + { + "name": "permission", + "type": "name" + } + ] + }, + { + "name": "key_weight", + "base": "", + "fields": [ + { + "name": "key", + "type": "public_key" + }, + { + "name": "weight", + "type": "uint16" + } + ] + }, + { + "name": "linkauth", + "base": "", + "fields": [ + { + "name": "account", + "type": "name" + }, + { + "name": "code", + "type": "name" + }, + { + "name": "type", + "type": "name" + }, + { + "name": "requirement", + "type": "name" + } + ] + }, + { + "name": "newaccount", + "base": "", + "fields": [ + { + "name": "creator", + "type": "name" + }, + { + "name": "name", + "type": "name" + }, + { + "name": "owner", + "type": "authority" + }, + { + "name": "active", + "type": "authority" + } + ] + }, + { + "name": "onerror", + "base": "", + "fields": [ + { + "name": "sender_id", + "type": "uint128" + }, + { + "name": "sent_trx", + "type": "bytes" + } + ] + }, + { + "name": "permission_level", + "base": "", + "fields": [ + { + "name": "actor", + "type": "name" + }, + { + "name": "permission", + "type": "name" + } + ] + }, + { + "name": "permission_level_weight", + "base": "", + "fields": [ + { + "name": "permission", + "type": "permission_level" + }, + { + "name": "weight", + "type": "uint16" + } + ] + }, + { + "name": "reqactivated", + "base": "", + "fields": [ + { + "name": "feature_digest", + "type": "checksum256" + } + ] + }, + { + "name": "setabi", + "base": "", + "fields": [ + { + "name": "account", + "type": "name" + }, + { + "name": "abi", + "type": "bytes" + } + ] + }, + { + "name": "setcode", + "base": "", + "fields": [ + { + "name": "account", + "type": "name" + }, + { + "name": "vmtype", + "type": "uint8" + }, + { + "name": "vmversion", + "type": "uint8" + }, + { + "name": "code", + "type": "bytes" + } + ] + }, + { + "name": "unlinkauth", + "base": "", + "fields": [ + { + "name": "account", + "type": "name" + }, + { + "name": "code", + "type": "name" + }, + { + "name": "type", + "type": "name" + } + ] + }, + { + "name": "updateauth", + "base": "", + "fields": [ + { + "name": "account", + "type": "name" + }, + { + "name": "permission", + "type": "name" + }, + { + "name": "parent", + "type": "name" + }, + { + "name": "auth", + "type": "authority" + } + ] + }, + { + "name": "wait_weight", + "base": "", + "fields": [ + { + "name": "wait_sec", + "type": "uint32" + }, + { + "name": "weight", + "type": "uint16" + } + ] + } + ], + "actions": [ + { + "name": "activate", + "type": "activate", + "ricardian_contract": "" + }, + { + "name": "canceldelay", + "type": "canceldelay", + "ricardian_contract": "" + }, + { + "name": "deleteauth", + "type": "deleteauth", + "ricardian_contract": "" + }, + { + "name": "linkauth", + "type": "linkauth", + "ricardian_contract": "" + }, + { + "name": "newaccount", + "type": "newaccount", + "ricardian_contract": "" + }, + { + "name": "onerror", + "type": "onerror", + "ricardian_contract": "" + }, + { + "name": "reqactivated", + "type": "reqactivated", + "ricardian_contract": "" + }, + { + "name": "setabi", + "type": "setabi", + "ricardian_contract": "" + }, + { + "name": "setcode", + "type": "setcode", + "ricardian_contract": "" + }, + { + "name": "unlinkauth", + "type": "unlinkauth", + "ricardian_contract": "" + }, + { + "name": "updateauth", + "type": "updateauth", + "ricardian_contract": "" + } + ], + "tables": [], + "kv_tables": {}, + "ricardian_clauses": [], + "variants": [], + "action_results": [] +} \ No newline at end of file diff --git a/contracts/contracts/eosio.boot/bin/eosio.boot.wasm b/contracts/contracts/eosio.boot/bin/eosio.boot.wasm new file mode 100755 index 00000000000..5efc287d4c7 Binary files /dev/null and b/contracts/contracts/eosio.boot/bin/eosio.boot.wasm differ diff --git a/contracts/contracts/eosio.boot/include/eosio.boot/eosio.boot.hpp b/contracts/contracts/eosio.boot/include/eosio.boot/eosio.boot.hpp new file mode 100644 index 00000000000..422c2e2df60 --- /dev/null +++ b/contracts/contracts/eosio.boot/include/eosio.boot/eosio.boot.hpp @@ -0,0 +1,260 @@ +#pragma once + +#include +#include + +namespace eosioboot { + + using eosio::action_wrapper; + using eosio::check; + using eosio::checksum256; + using eosio::ignore; + using eosio::name; + using eosio::permission_level; + using eosio::public_key; + + /** + * A weighted permission. + * + * @details Defines a weighted permission, that is a permission which has a weight associated. + * A permission is defined by an account name plus a permission name. The weight is going to be + * used against a threshold, if the weight is equal or greater than the threshold set then authorization + * will pass. + */ + struct permission_level_weight { + permission_level permission; + uint16_t weight; + + // explicit serialization macro is not necessary, used here only to improve compilation time + EOSLIB_SERIALIZE( permission_level_weight, (permission)(weight) ) + }; + + /** + * Weighted key. + * + * @details A weighted key is defined by a public key and an associated weight. + */ + struct key_weight { + eosio::public_key key; + uint16_t weight; + + // explicit serialization macro is not necessary, used here only to improve compilation time + EOSLIB_SERIALIZE( key_weight, (key)(weight) ) + }; + + /** + * Wait weight. + * + * @details A wait weight is defined by a number of seconds to wait for and a weight. + */ + struct wait_weight { + uint32_t wait_sec; + uint16_t weight; + + // explicit serialization macro is not necessary, used here only to improve compilation time + EOSLIB_SERIALIZE( wait_weight, (wait_sec)(weight) ) + }; + + /** + * Blockchain authority. + * + * @details An authority is defined by: + * - a vector of key_weights (a key_weight is a public key plus a weight), + * - a vector of permission_level_weights, (a permission_level is an account name plus a permission name) + * - a vector of wait_weights (a wait_weight is defined by a number of seconds to wait and a weight) + * - a threshold value + */ + struct authority { + uint32_t threshold = 0; + std::vector keys; + std::vector accounts; + std::vector waits; + + // explicit serialization macro is not necessary, used here only to improve compilation time + EOSLIB_SERIALIZE( authority, (threshold)(keys)(accounts)(waits) ) + }; + + /** + * @defgroup eosioboot eosio.boot + * @ingroup eosiocontracts + * + * eosio.boot is a extremely minimalistic system contract that only supports the native actions and an + * activate action that allows activating desired protocol features prior to deploying a system contract + * with more features such as eosio.bios or eosio.system. + * + * @{ + */ + class [[eosio::contract("eosio.boot")]] boot : public eosio::contract { + public: + using contract::contract; + /** + * @{ + * These actions map one-on-one with the ones defined in + * [Native Action Handlers](@ref native_action_handlers) section. + * They are present here so they can show up in the abi file and thus user can send them + * to this contract, but they have no specific implementation at this contract level, + * they will execute the implementation at the core level and nothing else. + */ + /** + * New account action + * + * @details Creates a new account. + * + * @param creator - the creator of the account + * @param name - the name of the new account + * @param owner - the authority for the owner permission of the new account + * @param active - the authority for the active permission of the new account + */ + [[eosio::action]] + void newaccount( name creator, + name name, + ignore owner, + ignore active) {} + /** + * Update authorization action. + * + * @details Updates pemission for an account. + * + * @param account - the account for which the permission is updated, + * @param pemission - the permission name which is updated, + * @param parem - the parent of the permission which is updated, + * @param aut - the json describing the permission authorization. + */ + [[eosio::action]] + void updateauth( ignore account, + ignore permission, + ignore parent, + ignore auth ) {} + + /** + * Delete authorization action. + * + * @details Deletes the authorization for an account's permision. + * + * @param account - the account for which the permission authorization is deleted, + * @param permission - the permission name been deleted. + */ + [[eosio::action]] + void deleteauth( ignore account, + ignore permission ) {} + + /** + * Link authorization action. + * + * @details Assigns a specific action from a contract to a permission you have created. Five system + * actions can not be linked `updateauth`, `deleteauth`, `linkauth`, `unlinkauth`, and `canceldelay`. + * This is useful because when doing authorization checks, the EOSIO based blockchain starts with the + * action needed to be authorized (and the contract belonging to), and looks up which permission + * is needed to pass authorization validation. If a link is set, that permission is used for authoraization + * validation otherwise then active is the default, with the exception of `eosio.any`. + * `eosio.any` is an implicit permission which exists on every account; you can link actions to `eosio.any` + * and that will make it so linked actions are accessible to any permissions defined for the account. + * + * @param account - the permission's owner to be linked and the payer of the RAM needed to store this link, + * @param code - the owner of the action to be linked, + * @param type - the action to be linked, + * @param requirement - the permission to be linked. + */ + [[eosio::action]] + void linkauth( ignore account, + ignore code, + ignore type, + ignore requirement ) {} + + /** + * Unlink authorization action. + * + * @details It's doing the reverse of linkauth action, by unlinking the given action. + * + * @param account - the owner of the permission to be unlinked and the receiver of the freed RAM, + * @param code - the owner of the action to be unlinked, + * @param type - the action to be unlinked. + */ + [[eosio::action]] + void unlinkauth( ignore account, + ignore code, + ignore type ) {} + + /** + * Cancel delay action. + * + * @details Cancels a deferred transaction. + * + * @param canceling_auth - the permission that authorizes this action, + * @param trx_id - the deferred transaction id to be cancelled. + */ + [[eosio::action]] + void canceldelay( ignore canceling_auth, ignore trx_id ) {} + + /** + * Set code action. + * + * @details Sets the contract code for an account. + * + * @param account - the account for which to set the contract code. + * @param vmtype - reserved, set it to zero. + * @param vmversion - reserved, set it to zero. + * @param code - the code content to be set, in the form of a blob binary.. + */ + [[eosio::action]] + void setcode( name account, uint8_t vmtype, uint8_t vmversion, const std::vector& code ) {} + + /** + * Set abi for contract. + * + * @details Set the abi for contract identified by `account` name. + * + * @param account - the name of the account to set the abi for + * @param abi - the abi hash represented as a vector of characters + */ + [[eosio::action]] + void setabi( name account, const std::vector& abi ) {} + + /** @}*/ + + /** + * On error action. + * + * @details Notification of this action is delivered to the sender of a deferred transaction + * when an objective error occurs while executing the deferred transaction. + * This action is not meant to be called directly. + * + * @param sender_id - the id for the deferred transaction chosen by the sender, + * @param sent_trx - the deferred transaction that failed. + */ + [[eosio::action]] + void onerror( ignore sender_id, ignore> sent_trx ); + + /** + * Activates a protocol feature. + * + * @details Activates a protocol feature + * + * @param feature_digest - hash of the protocol feature to activate. + */ + [[eosio::action]] + void activate( const eosio::checksum256& feature_digest ); + + /** + * Asserts that a protocol feature has been activated. + * + * @details Asserts that a protocol feature has been activated + * + * @param feature_digest - hash of the protocol feature to check for activation. + */ + [[eosio::action]] + void reqactivated( const eosio::checksum256& feature_digest ); + + using newaccount_action = action_wrapper<"newaccount"_n, &boot::newaccount>; + using updateauth_action = action_wrapper<"updateauth"_n, &boot::updateauth>; + using deleteauth_action = action_wrapper<"deleteauth"_n, &boot::deleteauth>; + using linkauth_action = action_wrapper<"linkauth"_n, &boot::linkauth>; + using unlinkauth_action = action_wrapper<"unlinkauth"_n, &boot::unlinkauth>; + using canceldelay_action = action_wrapper<"canceldelay"_n, &boot::canceldelay>; + using setcode_action = action_wrapper<"setcode"_n, &boot::setcode>; + using setabi_action = action_wrapper<"setabi"_n, &boot::setabi>; + using activate_action = action_wrapper<"activate"_n, &boot::activate>; + using reqactivated_action = action_wrapper<"reqactivated"_n, &boot::reqactivated>; + }; + /** @}*/ // end of @defgroup eosioboot eosio.boot +} /// namespace eosioboot diff --git a/contracts/contracts/eosio.boot/ricardian/eosio.boot.contracts.md.in b/contracts/contracts/eosio.boot/ricardian/eosio.boot.contracts.md.in new file mode 100644 index 00000000000..8a66a61dd01 --- /dev/null +++ b/contracts/contracts/eosio.boot/ricardian/eosio.boot.contracts.md.in @@ -0,0 +1,120 @@ +

activate

+ +--- +spec_version: "0.2.0" +title: Activate Protocol Feature +summary: 'Activate protocol feature {{nowrap feature_digest}}' +icon: @ICON_BASE_URL@/@ADMIN_ICON_URI@ +--- + +{{$action.account}} activates the protocol feature with a digest of {{feature_digest}}. + +

canceldelay

+ +--- +spec_version: "0.2.0" +title: Cancel Delayed Transaction +summary: '{{nowrap canceling_auth.actor}} cancels a delayed transaction' +icon: @ICON_BASE_URL@/@ACCOUNT_ICON_URI@ +--- + +{{canceling_auth.actor}} cancels the delayed transaction with id {{trx_id}}. + +

deleteauth

+ +--- +spec_version: "0.2.0" +title: Delete Account Permission +summary: 'Delete the {{nowrap permission}} permission of {{nowrap account}}' +icon: @ICON_BASE_URL@/@ACCOUNT_ICON_URI@ +--- + +Delete the {{permission}} permission of {{account}}. + +

linkauth

+ +--- +spec_version: "0.2.0" +title: Link Action to Permission +summary: '{{nowrap account}} sets the minimum required permission for the {{#if type}}{{nowrap type}} action of the{{/if}} {{nowrap code}} contract to {{nowrap requirement}}' +icon: @ICON_BASE_URL@/@ACCOUNT_ICON_URI@ +--- + +{{account}} sets the minimum required permission for the {{#if type}}{{type}} action of the{{/if}} {{code}} contract to {{requirement}}. + +{{#if type}}{{else}}Any links explicitly associated to specific actions of {{code}} will take precedence.{{/if}} + +

newaccount

+ +--- +spec_version: "0.2.0" +title: Create New Account +summary: '{{nowrap creator}} creates a new account with the name {{nowrap name}}' +icon: @ICON_BASE_URL@/@ACCOUNT_ICON_URI@ +--- + +{{creator}} creates a new account with the name {{name}} and the following permissions: + +owner permission with authority: +{{to_json owner}} + +active permission with authority: +{{to_json active}} + +

reqactivated

+ +--- +spec_version: "0.2.0" +title: Assert Protocol Feature Activation +summary: 'Assert that protocol feature {{nowrap feature_digest}} has been activated' +icon: @ICON_BASE_URL@/@ADMIN_ICON_URI@ +--- + +Assert that the protocol feature with a digest of {{feature_digest}} has been activated. + +

setabi

+ +--- +spec_version: "0.2.0" +title: Deploy Contract ABI +summary: 'Deploy contract ABI on account {{nowrap account}}' +icon: @ICON_BASE_URL@/@ACCOUNT_ICON_URI@ +--- + +Deploy the ABI file associated with the contract on account {{account}}. + +

setcode

+ +--- +spec_version: "0.2.0" +title: Deploy Contract Code +summary: 'Deploy contract code on account {{nowrap account}}' +icon: @ICON_BASE_URL@/@ACCOUNT_ICON_URI@ +--- + +Deploy compiled contract code to the account {{account}}. + +

unlinkauth

+ +--- +spec_version: "0.2.0" +title: Unlink Action from Permission +summary: '{{nowrap account}} unsets the minimum required permission for the {{#if type}}{{nowrap type}} action of the{{/if}} {{nowrap code}} contract' +icon: @ICON_BASE_URL@/@ACCOUNT_ICON_URI@ +--- + +{{account}} removes the association between the {{#if type}}{{type}} action of the{{/if}} {{code}} contract and its minimum required permission. + +{{#if type}}{{else}}This will not remove any links explicitly associated to specific actions of {{code}}.{{/if}} + +

updateauth

+ +--- +spec_version: "0.2.0" +title: Modify Account Permission +summary: 'Add or update the {{nowrap permission}} permission of {{nowrap account}}' +icon: @ICON_BASE_URL@/@ACCOUNT_ICON_URI@ +--- + +Modify, and create if necessary, the {{permission}} permission of {{account}} to have a parent permission of {{parent}} and the following authority: +{{to_json auth}} diff --git a/contracts/contracts/eosio.boot/src/eosio.boot.cpp b/contracts/contracts/eosio.boot/src/eosio.boot.cpp new file mode 100644 index 00000000000..03ec928da63 --- /dev/null +++ b/contracts/contracts/eosio.boot/src/eosio.boot.cpp @@ -0,0 +1,19 @@ +#include +#include + +namespace eosioboot { + +void boot::onerror( ignore, ignore> ) { + check( false, "the onerror action cannot be called directly" ); +} + +void boot::activate( const eosio::checksum256& feature_digest ) { + require_auth( get_self() ); + eosio::preactivate_feature( feature_digest ); +} + +void boot::reqactivated( const eosio::checksum256& feature_digest ) { + check( eosio::is_feature_activated( feature_digest ), "protocol feature is not activated" ); +} + +} diff --git a/contracts/enable-kv/README.md b/contracts/enable-kv/README.md new file mode 100644 index 00000000000..68634ab5307 --- /dev/null +++ b/contracts/enable-kv/README.md @@ -0,0 +1,22 @@ +## Summary +This README illustrates the steps for usage of enable-kv.sh script with kv_map as an example. Set your environment variables as follows before beginning the rest of the steps. +1. export EOS_2_1_0=[eos 2.1.0 directory] +1. export EOSIO_CDT_1_8_0=[eosio.cdt 1.8.0 directory] +1. export PATH=$EOS_2_1_0/build/bin:$PATH + +## Steps +Bring up nodeos in a different terminal +1. export PATH=$EOS_2_1_0/build/bin:$PATH +1. nodeos -e -p eosio --plugin eosio::producer_plugin --plugin eosio::producer_api_plugin --plugin eosio::chain_api_plugin --plugin eosio::http_plugin --plugin eosio::history_plugin --plugin eosio::history_api_plugin --filter-on=* --access-control-allow-origin=* --contracts-console --http-validate-host=false --verbose-http-errors --max-transaction-time=1000 --backing-store chainbase --data-dir=datadir + +In the first terminal +1. $EOS_2_1_0/contracts/enable-kv/enable-kv.sh -c $EOS_2_1_0/build/contracts/contracts/ +1. cd $EOSIO_CDT_1_8_0/examples/kv_map +1. mkdir build +1. cd build +1. cmake .. -DCMAKE_PREFIX_PATH=$EOS_2_1_0/build +1. make +1. cleos create account eosio jane EOS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV +1. cleos set contract jane $EOSIO_CDT_1_8_0/examples/kv_map/build/kv_map -p jane@active +1. cleos push action jane upsert '[1, "jane.acct" "jane", "doe", "1 main st", "new york", "NY", "USA", "123"]' -p jane@active +1. cleos push action jane get '[1]' -p jane@active \ No newline at end of file diff --git a/contracts/enable-kv/enable-kv.sh b/contracts/enable-kv/enable-kv.sh new file mode 100755 index 00000000000..5980d734f45 --- /dev/null +++ b/contracts/enable-kv/enable-kv.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +usage() +{ + echo "usage: enable-kv.sh -c [-u ]" +} + +while [ "$1" != "" ]; do + case $1 in + -c | --contracts-dir ) shift + contracts_dir=$1 + ;; + -u | --url ) shift + url=$1 + ;; + * ) usage + exit 1 + esac + shift +done + +if [ "$contracts_dir" = "" ]; then + usage + exit 1 +fi + +if [ "$url" = "" ]; then + url=http://127.0.0.1:8888 +fi + +curl -X POST $url/v1/producer/schedule_protocol_feature_activations -d '{"protocol_features_to_activate": ["0ec7e080177b2c02b278d5088611686b49d739925a92d9bfcacd7fc6b74053bd"]}' +sleep 1s +cleos set contract eosio $contracts_dir/eosio.boot -p eosio@active +sleep 1s +sleep 1s +sleep 1s +cleos push action eosio activate '["299dcb6af692324b899b39f16d5a530a33062804e41f09dc97e9f156b4476707"]' -p eosio@active +sleep 1s +sleep 1s + +# KV_DATABASE +cleos push action eosio activate '["825ee6288fb1373eab1b5187ec2f04f6eacb39cb3a97f356a07c91622dd61d16"]' -p eosio@active +# ACTION_RETURN_VALUE +cleos push action eosio activate '["c3a6138c5061cf291310887c0b5c71fcaffeab90d5deb50d3b9e687cead45071"]' -p eosio@active +# RAM_RESTRICTIONS +cleos push action eosio activate '["4e7bf348da00a945489b2a681749eb56f5de00b900014e137ddae39f48f69d67"]' -p eosio@active +# GET_SENDER +cleos push action eosio activate '["f0af56d2c5a48d60a4a5b5c903edfb7db3a736a94ed589d0b797df33ff9d3e1d"]' -p eosio@active +# FORWARD_SETCODE +cleos push action eosio activate '["2652f5f96006294109b3dd0bbde63693f55324af452b799ee137a81a905eed25"]' -p eosio@active +# ONLY_BILL_FIRST_AUTHORIZER +cleos push action eosio activate '["8ba52fe7a3956c5cd3a656a3174b931d3bb2abb45578befc59f283ecd816a405"]' -p eosio@active +# RESTRICT_ACTION_TO_SELF +cleos push action eosio activate '["ad9e3d8f650687709fd68f4b90b41f7d825a365b02c23a636cef88ac2ac00c43"]' -p eosio@active +# DISALLOW_EMPTY_PRODUCER_SCHEDULE +cleos push action eosio activate '["68dcaa34c0517d19666e6b33add67351d8c5f69e999ca1e37931bc410a297428"]' -p eosio@active +# FIX_LINKAUTH_RESTRICTION +cleos push action eosio activate '["e0fb64b1085cc5538970158d05a009c24e276fb94e1a0bf6a528b48fbc4ff526"]' -p eosio@active +# REPLACE_DEFERRED +cleos push action eosio activate '["ef43112c6543b88db2283a2e077278c315ae2c84719a8b25f25cc88565fbea99"]' -p eosio@active +# NO_DUPLICATE_DEFERRED_ID +cleos push action eosio activate '["4a90c00d55454dc5b059055ca213579c6ea856967712a56017487886a4d4cc0f"]' -p eosio@active +# ONLY_LINK_TO_EXISTING_PERMISSION +cleos push action eosio activate '["1a99a59d87e06e09ec5b028a9cbb7749b4a5ad8819004365d02dc4379a8b7241"]' -p eosio@active +# CONFIGURABLE_WASM_LIMITS +cleos push action eosio activate '["bf61537fd21c61a60e542a5d66c3f6a78da0589336868307f94a82bccea84e88"]' -p eosio@active +# BLOCKCHAIN_PARAMETERS +cleos push action eosio activate '["5443fcf88330c586bc0e5f3dee10e7f63c76c00249c87fe4fbf7f38c082006b4"]' -p eosio@active + +sleep 1s +cleos set contract eosio $contracts_dir/eosio.bios -p eosio@active +sleep 1s +cleos push action eosio setkvparams '[{"max_key_size":64, "max_value_size":1024, "max_iterators":128}]' -p eosio@active diff --git a/docs.json b/docs.json index 698c666cecb..fa00989b863 100644 --- a/docs.json +++ b/docs.json @@ -43,15 +43,6 @@ "disable_summary_gen": true } }, - { - "name": "swagger", - "options": { - "swagger_path": "plugins/test_control_api_plugin/test_control.swagger.yaml", - "swagger_dest_path": "nodeos/plugins/test_control_api_plugin/api-reference", - "disable_filters": true, - "disable_summary_gen": true - } - }, { "name": "swagger", "options": { @@ -60,6 +51,18 @@ "disable_filters": true, "disable_summary_gen": true } + }, + { + "name": "doxygen_to_xml", + "options": { + "INPUT": "libraries/chain/include/eosio/chain/webassembly/interface.hpp plugins/blockvault_client_plugin/include/eosio/blockvault_client_plugin/blockvault.hpp" + }, + "disable_default_filters" :true, + "filters" : [] + }, + { + "name": "doxybook", + "options": {} } ] } diff --git a/docs/00_install/00_install-prebuilt-binaries.md b/docs/00_install/00_install-prebuilt-binaries.md index d9e51f95174..856e43a485a 100644 --- a/docs/00_install/00_install-prebuilt-binaries.md +++ b/docs/00_install/00_install-prebuilt-binaries.md @@ -22,16 +22,20 @@ brew remove eosio ``` ### Ubuntu Linux: - +#### Ubuntu 20.04 Package Install +```sh +wget https://github.com/eosio/eos/releases/download/v2.1.0/eosio_2.1.0-1-ubuntu-20.04_amd64.deb +sudo apt install ./eosio_2.1.0-1-ubuntu-20.04_amd64.deb +``` #### Ubuntu 18.04 Package Install ```sh -wget https://github.com/eosio/eos/releases/download/v2.0.12/eosio_2.0.12-1-ubuntu-18.04_amd64.deb -sudo apt install ./eosio_2.0.12-1-ubuntu-18.04_amd64.deb +wget https://github.com/eosio/eos/releases/download/v2.1.0/eosio_2.1.0-1-ubuntu-18.04_amd64.deb +sudo apt install ./eosio_2.1.0-1-ubuntu-18.04_amd64.deb ``` #### Ubuntu 16.04 Package Install ```sh -wget https://github.com/eosio/eos/releases/download/v2.0.12/eosio_2.0.12-1-ubuntu-16.04_amd64.deb -sudo apt install ./eosio_2.0.12-1-ubuntu-16.04_amd64.deb +wget https://github.com/eosio/eos/releases/download/v2.1.0/eosio_2.1.0-1-ubuntu-16.04_amd64.deb +sudo apt install ./eosio_2.1.0-1-ubuntu-16.04_amd64.deb ``` #### Ubuntu Package Uninstall ```sh @@ -40,10 +44,15 @@ sudo apt remove eosio ### RPM-based (CentOS, Amazon Linux, etc.): -#### RPM Package Install +#### RPM Package Install CentOS 7 +```sh +wget https://github.com/eosio/eos/releases/download/v2.1.0/eosio-2.1.0-1.el7.x86_64.rpm +sudo yum install ./eosio-2.1.0-1.el7.x86_64.rpm +``` +#### RPM Package Install CentOS 8 ```sh -wget https://github.com/eosio/eos/releases/download/v2.0.12/eosio-2.0.12-1.el7.x86_64.rpm -sudo yum install ./eosio-2.0.12-1.el7.x86_64.rpm +wget https://github.com/eosio/eos/releases/download/v2.1.0/eosio-2.1.0-1.el8.x86_64.rpm +sudo yum install ./eosio-2.1.0-1.el8.x86_64.rpm ``` #### RPM Package Uninstall ```sh diff --git a/docs/00_install/01_build-from-source/02_manual-build/03_platforms/amazon_linux-2.md b/docs/00_install/01_build-from-source/02_manual-build/03_platforms/amazon_linux-2.md index dbc27799114..21d69ed9505 100644 --- a/docs/00_install/01_build-from-source/02_manual-build/03_platforms/amazon_linux-2.md +++ b/docs/00_install/01_build-from-source/02_manual-build/03_platforms/amazon_linux-2.md @@ -57,29 +57,6 @@ cd $EOSIO_INSTALL_LOCATION && curl -LO https://boostorg.jfrog.io/artifactory/mai ./bootstrap.sh --prefix=$EOSIO_INSTALL_LOCATION && \ ./b2 --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install && \ rm -rf $EOSIO_INSTALL_LOCATION/boost_1_71_0.tar.bz2 $EOSIO_INSTALL_LOCATION/boost_1_71_0 -# build mongodb -cd $EOSIO_INSTALL_LOCATION && curl -LO https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-amazon-3.6.3.tgz && \ - tar -xzf mongodb-linux-x86_64-amazon-3.6.3.tgz && rm -f mongodb-linux-x86_64-amazon-3.6.3.tgz && \ - mv $EOSIO_INSTALL_LOCATION/mongodb-linux-x86_64-amazon-3.6.3/bin/* $EOSIO_INSTALL_LOCATION/bin/ && \ - rm -rf $EOSIO_INSTALL_LOCATION/mongodb-linux-x86_64-amazon-3.6.3 -# build mongodb c driver -cd $EOSIO_INSTALL_LOCATION && curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz && \ - tar -xzf mongo-c-driver-1.13.0.tar.gz && cd mongo-c-driver-1.13.0 && \ - mkdir -p build && cd build && \ - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$EOSIO_INSTALL_LOCATION -DENABLE_BSON=ON -DENABLE_SSL=OPENSSL -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON -DENABLE_ICU=OFF -DENABLE_SNAPPY=OFF .. && \ - make -j$(nproc) && \ - make install && \ - rm -rf $EOSIO_INSTALL_LOCATION/mongo-c-driver-1.13.0.tar.gz $EOSIO_INSTALL_LOCATION/mongo-c-driver-1.13.0 -# build mongodb cxx driver -cd $EOSIO_INSTALL_LOCATION && curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz && \ - tar -xzf mongo-cxx-driver-r3.4.0.tar.gz && cd mongo-cxx-driver-r3.4.0 && \ - sed -i 's/\"maxAwaitTimeMS\", count/\"maxAwaitTimeMS\", static_cast(count)/' src/mongocxx/options/change_stream.cpp && \ - sed -i 's/add_subdirectory(test)//' src/mongocxx/CMakeLists.txt src/bsoncxx/CMakeLists.txt && \ - mkdir -p build && cd build && \ - cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$EOSIO_INSTALL_LOCATION .. && \ - make -j$(nproc) && \ - make install && \ - rm -rf $EOSIO_INSTALL_LOCATION/mongo-cxx-driver-r3.4.0.tar.gz $EOSIO_INSTALL_LOCATION/mongo-cxx-driver-r3.4.0 ``` ## Build EOSIO @@ -91,7 +68,7 @@ These commands build the EOSIO software on the specified OS. Make sure to [Insta ```sh export EOSIO_BUILD_LOCATION=$EOSIO_LOCATION/build mkdir -p $EOSIO_BUILD_LOCATION -cd $EOSIO_BUILD_LOCATION && $EOSIO_INSTALL_LOCATION/bin/cmake -DCMAKE_BUILD_TYPE='Release' -DCMAKE_CXX_COMPILER='clang++' -DCMAKE_C_COMPILER='clang' -DCMAKE_INSTALL_PREFIX=$EOSIO_INSTALL_LOCATION -DBUILD_MONGO_DB_PLUGIN=true $EOSIO_LOCATION +cd $EOSIO_BUILD_LOCATION && $EOSIO_INSTALL_LOCATION/bin/cmake -DCMAKE_BUILD_TYPE='Release' -DCMAKE_CXX_COMPILER='clang++' -DCMAKE_C_COMPILER='clang' -DCMAKE_INSTALL_PREFIX=$EOSIO_INSTALL_LOCATION $EOSIO_LOCATION cd $EOSIO_BUILD_LOCATION && make -j$(nproc) ``` @@ -104,7 +81,6 @@ cd $EOSIO_BUILD_LOCATION && make install ## Test EOSIO These commands validate the EOSIO software installation on the specified OS. This task is optional but recommended. Make sure to [Install EOSIO](#install-eosio) first. ```sh -$EOSIO_INSTALL_LOCATION/bin/mongod --fork --logpath $(pwd)/mongod.log --dbpath $(pwd)/mongodata cd $EOSIO_BUILD_LOCATION && make test ``` diff --git a/docs/00_install/01_build-from-source/02_manual-build/03_platforms/centos-7.7.md b/docs/00_install/01_build-from-source/02_manual-build/03_platforms/centos-7.7.md index 8f4222d3a6c..8a7fdbd5aec 100644 --- a/docs/00_install/01_build-from-source/02_manual-build/03_platforms/centos-7.7.md +++ b/docs/00_install/01_build-from-source/02_manual-build/03_platforms/centos-7.7.md @@ -42,7 +42,7 @@ yum update -y && \ yum --enablerepo=extras install -y centos-release-scl && \ yum --enablerepo=extras install -y devtoolset-8 && \ yum --enablerepo=extras install -y which git autoconf automake libtool make bzip2 doxygen \ - graphviz bzip2-devel openssl-devel gmp-devel ocaml libicu-devel \ + graphviz bzip2-devel openssl-devel gmp-devel ocaml \ python python-devel rh-python36 file libusbx-devel \ libcurl-devel patch vim-common jq llvm-toolset-7.0-llvm-devel llvm-toolset-7.0-llvm-static # build cmake @@ -65,31 +65,6 @@ cd $EOSIO_INSTALL_LOCATION && curl -LO https://boostorg.jfrog.io/artifactory/mai ./bootstrap.sh --prefix=$EOSIO_INSTALL_LOCATION && \ ./b2 --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install && \ rm -rf $EOSIO_INSTALL_LOCATION/boost_1_71_0.tar.bz2 $EOSIO_INSTALL_LOCATION/boost_1_71_0 -# build mongodb -cd $EOSIO_INSTALL_LOCATION && curl -LO https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-amazon-3.6.3.tgz && \ - tar -xzf mongodb-linux-x86_64-amazon-3.6.3.tgz && rm -f mongodb-linux-x86_64-amazon-3.6.3.tgz && \ - mv $EOSIO_INSTALL_LOCATION/mongodb-linux-x86_64-amazon-3.6.3/bin/* $EOSIO_INSTALL_LOCATION/bin/ && \ - rm -rf $EOSIO_INSTALL_LOCATION/mongodb-linux-x86_64-amazon-3.6.3 -# build mongodb c driver -cd $EOSIO_INSTALL_LOCATION && curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz && \ - source /opt/rh/devtoolset-8/enable && \ - tar -xzf mongo-c-driver-1.13.0.tar.gz && cd mongo-c-driver-1.13.0 && \ - mkdir -p build && cd build && \ - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$EOSIO_INSTALL_LOCATION -DENABLE_BSON=ON -DENABLE_SSL=OPENSSL -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON -DENABLE_ICU=OFF -DENABLE_SNAPPY=OFF .. && \ - make -j$(nproc) && \ - make install && \ - rm -rf $EOSIO_INSTALL_LOCATION/mongo-c-driver-1.13.0.tar.gz $EOSIO_INSTALL_LOCATION/mongo-c-driver-1.13.0 -# build mongodb cxx driver -cd $EOSIO_INSTALL_LOCATION && curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz && \ - source /opt/rh/devtoolset-8/enable && \ - tar -xzf mongo-cxx-driver-r3.4.0.tar.gz && cd mongo-cxx-driver-r3.4.0 && \ - sed -i 's/\"maxAwaitTimeMS\", count/\"maxAwaitTimeMS\", static_cast(count)/' src/mongocxx/options/change_stream.cpp && \ - sed -i 's/add_subdirectory(test)//' src/mongocxx/CMakeLists.txt src/bsoncxx/CMakeLists.txt && \ - mkdir -p build && cd build && \ - cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$EOSIO_INSTALL_LOCATION .. && \ - make -j$(nproc) && \ - make install && \ - rm -rf $EOSIO_INSTALL_LOCATION/mongo-cxx-driver-r3.4.0.tar.gz $EOSIO_INSTALL_LOCATION/mongo-cxx-driver-r3.4.0 ``` ## Build EOSIO @@ -101,7 +76,7 @@ These commands build the EOSIO software on the specified OS. Make sure to [Insta ```sh export EOSIO_BUILD_LOCATION=$EOSIO_LOCATION/build mkdir -p $EOSIO_BUILD_LOCATION -cd $EOSIO_BUILD_LOCATION && source /opt/rh/devtoolset-8/enable && cmake -DCMAKE_BUILD_TYPE='Release' -DLLVM_DIR='/opt/rh/llvm-toolset-7.0/root/usr/lib64/cmake/llvm' -DCMAKE_INSTALL_PREFIX=$EOSIO_INSTALL_LOCATION -DBUILD_MONGO_DB_PLUGIN=true $EOSIO_LOCATION +cd $EOSIO_BUILD_LOCATION && source /opt/rh/devtoolset-8/enable && cmake -DCMAKE_BUILD_TYPE='Release' -DLLVM_DIR='/opt/rh/llvm-toolset-7.0/root/usr/lib64/cmake/llvm' -DCMAKE_INSTALL_PREFIX=$EOSIO_INSTALL_LOCATION $EOSIO_LOCATION cd $EOSIO_BUILD_LOCATION && make -j$(nproc) ``` @@ -114,7 +89,6 @@ cd $EOSIO_BUILD_LOCATION && make install ## Test EOSIO These commands validate the EOSIO software installation on the specified OS. This task is optional but recommended. Make sure to [Install EOSIO](#install-eosio) first. ```sh -$EOSIO_INSTALL_LOCATION/bin/mongod --fork --logpath $(pwd)/mongod.log --dbpath $(pwd)/mongodata cd $EOSIO_BUILD_LOCATION && source /opt/rh/rh-python36/enable && make test ``` diff --git a/docs/00_install/01_build-from-source/02_manual-build/03_platforms/macos-10.14.md b/docs/00_install/01_build-from-source/02_manual-build/03_platforms/macos-10.14.md index 55bfe48afc5..15e58cc1064 100644 --- a/docs/00_install/01_build-from-source/02_manual-build/03_platforms/macos-10.14.md +++ b/docs/00_install/01_build-from-source/02_manual-build/03_platforms/macos-10.14.md @@ -39,27 +39,6 @@ These commands install the EOSIO software dependencies. Make sure to [Download # install dependencies brew install cmake python libtool libusb graphviz automake wget gmp pkgconfig doxygen openssl@1.1 jq boost || : export PATH=$EOSIO_INSTALL_LOCATION/bin:$PATH -# install mongodb -mkdir -p $EOSIO_INSTALL_LOCATION/bin -cd $EOSIO_INSTALL_LOCATION && curl -OL https://fastdl.mongodb.org/osx/mongodb-osx-ssl-x86_64-3.6.3.tgz - tar -xzf mongodb-osx-ssl-x86_64-3.6.3.tgz && rm -f mongodb-osx-ssl-x86_64-3.6.3.tgz && \ - mv $EOSIO_INSTALL_LOCATION/mongodb-osx-x86_64-3.6.3/bin/* $EOSIO_INSTALL_LOCATION/bin/ && \ - rm -rf $EOSIO_INSTALL_LOCATION/mongodb-osx-x86_64-3.6.3 && rm -rf $EOSIO_INSTALL_LOCATION/mongodb-osx-ssl-x86_64-3.6.3.tgz -# install mongo-c-driver from source -cd $EOSIO_INSTALL_LOCATION && curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz && \ - tar -xzf mongo-c-driver-1.13.0.tar.gz && cd mongo-c-driver-1.13.0 && \ - mkdir -p cmake-build && cd cmake-build && \ - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$EOSIO_INSTALL_LOCATION -DENABLE_BSON=ON -DENABLE_SSL=DARWIN -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON -DENABLE_ICU=OFF -DENABLE_SASL=OFF -DENABLE_SNAPPY=OFF .. && \ - make -j $(getconf _NPROCESSORS_ONLN) && \ - make install && \ - rm -rf $EOSIO_INSTALL_LOCATION/mongo-c-driver-1.13.0.tar.gz $EOSIO_INSTALL_LOCATION/mongo-c-driver-1.13.0 -# install mongo-cxx-driver from source -cd $EOSIO_INSTALL_LOCATION && curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz && \ - tar -xzf mongo-cxx-driver-r3.4.0.tar.gz && cd mongo-cxx-driver-r3.4.0/build && \ - cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$EOSIO_INSTALL_LOCATION .. && \ - make -j $(getconf _NPROCESSORS_ONLN) VERBOSE=1 && \ - make install && \ - rm -rf $EOSIO_INSTALL_LOCATION/mongo-cxx-driver-r3.4.0.tar.gz $EOSIO_INSTALL_LOCATION/mongo-cxx-driver-r3.4.0 ``` ## Build EOSIO @@ -71,7 +50,7 @@ These commands build the EOSIO software on the specified OS. Make sure to [Insta ```sh export EOSIO_BUILD_LOCATION=$EOSIO_LOCATION/build mkdir -p $EOSIO_BUILD_LOCATION -cd $EOSIO_BUILD_LOCATION && cmake -DCMAKE_BUILD_TYPE='Release' -DCMAKE_INSTALL_PREFIX=$EOSIO_INSTALL_LOCATION -DBUILD_MONGO_DB_PLUGIN=true $EOSIO_LOCATION +cd $EOSIO_BUILD_LOCATION && cmake -DCMAKE_BUILD_TYPE='Release' -DCMAKE_INSTALL_PREFIX=$EOSIO_INSTALL_LOCATION $EOSIO_LOCATION cd $EOSIO_BUILD_LOCATION && make -j$(getconf _NPROCESSORS_ONLN) ``` @@ -84,7 +63,6 @@ cd $EOSIO_BUILD_LOCATION && make install ## Test EOSIO These commands validate the EOSIO software installation on the specified OS. This task is optional but recommended. Make sure to [Install EOSIO](#install-eosio) first. ```sh -$EOSIO_INSTALL_LOCATION/bin/mongod --fork --logpath $(pwd)/mongod.log --dbpath $(pwd)/mongodata cd $EOSIO_BUILD_LOCATION && make test ``` diff --git a/docs/00_install/01_build-from-source/02_manual-build/03_platforms/ubuntu-18.04.md b/docs/00_install/01_build-from-source/02_manual-build/03_platforms/ubuntu-18.04.md index 31659865aa7..49717b5f101 100644 --- a/docs/00_install/01_build-from-source/02_manual-build/03_platforms/ubuntu-18.04.md +++ b/docs/00_install/01_build-from-source/02_manual-build/03_platforms/ubuntu-18.04.md @@ -38,7 +38,7 @@ These commands install the EOSIO software dependencies. Make sure to [Download ```sh # install dependencies apt-get install -y make bzip2 automake libbz2-dev libssl-dev doxygen graphviz libgmp3-dev \ - autotools-dev libicu-dev python2.7 python2.7-dev python3 python3-dev \ + autotools-dev python2.7 python2.7-dev python3 python3-dev \ autoconf libtool curl zlib1g-dev sudo ruby libusb-1.0-0-dev \ libcurl4-gnutls-dev pkg-config patch llvm-7-dev clang-7 vim-common jq # build cmake @@ -57,29 +57,6 @@ cd $EOSIO_INSTALL_LOCATION && curl -LO https://boostorg.jfrog.io/artifactory/mai ./bootstrap.sh --prefix=$EOSIO_INSTALL_LOCATION && \ ./b2 --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install && \ rm -rf $EOSIO_INSTALL_LOCATION/boost_1_71_0.tar.bz2 $EOSIO_INSTALL_LOCATION/boost_1_71_0 -# build mongodb -cd $EOSIO_INSTALL_LOCATION && curl -LO https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1804-4.1.1.tgz && \ - tar -xzf mongodb-linux-x86_64-ubuntu1804-4.1.1.tgz && rm -f mongodb-linux-x86_64-ubuntu1804-4.1.1.tgz && \ - mv $EOSIO_INSTALL_LOCATION/mongodb-linux-x86_64-ubuntu1804-4.1.1/bin/* $EOSIO_INSTALL_LOCATION/bin/ && \ - rm -rf $EOSIO_INSTALL_LOCATION/mongodb-linux-x86_64-ubuntu1804-4.1.1 -# build mongodb c driver -cd $EOSIO_INSTALL_LOCATION && curl -LO https://github.com/mongodb/mongo-c-driver/releases/download/1.13.0/mongo-c-driver-1.13.0.tar.gz && \ - tar -xzf mongo-c-driver-1.13.0.tar.gz && cd mongo-c-driver-1.13.0 && \ - mkdir -p build && cd build && \ - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$EOSIO_INSTALL_LOCATION -DENABLE_BSON=ON -DENABLE_SSL=OPENSSL -DENABLE_AUTOMATIC_INIT_AND_CLEANUP=OFF -DENABLE_STATIC=ON -DENABLE_ICU=OFF -DENABLE_SNAPPY=OFF .. && \ - make -j$(nproc) && \ - make install && \ - rm -rf $EOSIO_INSTALL_LOCATION/mongo-c-driver-1.13.0.tar.gz $EOSIO_INSTALL_LOCATION/mongo-c-driver-1.13.0 -# build mongodb cxx driver -cd $EOSIO_INSTALL_LOCATION && curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o mongo-cxx-driver-r3.4.0.tar.gz && \ - tar -xzf mongo-cxx-driver-r3.4.0.tar.gz && cd mongo-cxx-driver-r3.4.0 && \ - sed -i 's/\"maxAwaitTimeMS\", count/\"maxAwaitTimeMS\", static_cast(count)/' src/mongocxx/options/change_stream.cpp && \ - sed -i 's/add_subdirectory(test)//' src/mongocxx/CMakeLists.txt src/bsoncxx/CMakeLists.txt && \ - mkdir -p build && cd build && \ - cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$EOSIO_INSTALL_LOCATION .. && \ - make -j$(nproc) && \ - make install && \ - rm -rf $EOSIO_INSTALL_LOCATION/mongo-cxx-driver-r3.4.0.tar.gz $EOSIO_INSTALL_LOCATION/mongo-cxx-driver-r3.4.0 ``` ## Build EOSIO @@ -91,7 +68,7 @@ These commands build the EOSIO software on the specified OS. Make sure to [Insta ```sh export EOSIO_BUILD_LOCATION=$EOSIO_LOCATION/build mkdir -p $EOSIO_BUILD_LOCATION -cd $EOSIO_BUILD_LOCATION && cmake -DCMAKE_BUILD_TYPE='Release' -DCMAKE_CXX_COMPILER='clang++-7' -DCMAKE_C_COMPILER='clang-7' -DLLVM_DIR='/usr/lib/llvm-7/lib/cmake/llvm' -DCMAKE_INSTALL_PREFIX=$EOSIO_INSTALL_LOCATION -DBUILD_MONGO_DB_PLUGIN=true $EOSIO_LOCATION +cd $EOSIO_BUILD_LOCATION && cmake -DCMAKE_BUILD_TYPE='Release' -DCMAKE_CXX_COMPILER='clang++-7' -DCMAKE_C_COMPILER='clang-7' -DLLVM_DIR='/usr/lib/llvm-7/lib/cmake/llvm' -DCMAKE_INSTALL_PREFIX=$EOSIO_INSTALL_LOCATION $EOSIO_LOCATION cd $EOSIO_BUILD_LOCATION && make -j$(nproc) ``` @@ -104,7 +81,6 @@ cd $EOSIO_BUILD_LOCATION && make install ## Test EOSIO These commands validate the EOSIO software installation on the specified OS. Make sure to [Install EOSIO](#install-eosio) first. (**Note**: This task is optional but recommended.) ```sh -$EOSIO_INSTALL_LOCATION/bin/mongod --fork --logpath $(pwd)/mongod.log --dbpath $(pwd)/mongodata cd $EOSIO_BUILD_LOCATION && make test ``` diff --git a/docs/00_install/index.md b/docs/00_install/index.md index 3153b2547c6..517a6d7a3b8 100644 --- a/docs/00_install/index.md +++ b/docs/00_install/index.md @@ -12,13 +12,18 @@ There are various ways to install and use the EOSIO software: ## Supported Operating Systems -EOSIO currently supports the following operating systems: +The EOSIO software supports the following environments for development and/or deployment: -1. Amazon Linux 2 -2. CentOS 7 -3. Ubuntu 16.04 -4. Ubuntu 18.04 -5. MacOS 10.14 (Mojave) +**Linux Distributions** +* Amazon Linux 2 +* CentOS Linux 8.x +* CentOS Linux 7.x +* Ubuntu 20.04 +* Ubuntu 18.04 +* Ubuntu 16.04 + +**macOS** +* macOS 10.14 (Mojave) or later [[info | Note]] | It may be possible to install EOSIO on other Unix-based operating systems. This is not officially supported, though. diff --git a/docs/01_nodeos/02_usage/03_development-environment/00_local-single-node-testnet.md b/docs/01_nodeos/02_usage/03_development-environment/00_local-single-node-testnet.md index 53ca22048c4..e5e1bceae9c 100644 --- a/docs/01_nodeos/02_usage/03_development-environment/00_local-single-node-testnet.md +++ b/docs/01_nodeos/02_usage/03_development-environment/00_local-single-node-testnet.md @@ -8,7 +8,7 @@ This section describes how to set up a single-node blockchain configuration runn ![Single host single node testnet](single-host-single-node-testnet.png) -`cleos` is used to manage the wallets, manage the accounts, and invoke actions on the blockchain. `keosd` performs wallet management, including digital signing. If not started explicitly, `keosd` is started by `cleos` by default. +`cleos` is used to interact with wallets, and send actions to the blockchain. `keosd` performs wallet management, including digital signing. If not started explicitly, `keosd` is started by `cleos` by default. ## Before you begin @@ -115,8 +115,9 @@ nodeos * Mac OS: `~/Library/Application\ Support/eosio/nodeos/data` * Linux: `~/.local/share/eosio/nodeos/data` - + A data folder can be specified using the `--data-dir` command line argument to `nodeos`. [[info | What's next?]] -| We will explore how to setup and run a [single-host, multi-node testnet](01_local-multi-node-testnet.md). +| You can explore how to setup and run a [single-host, single-node testnet with consensus](10_local-single-node-testnet-consensus.md) or +| You can explore how to setup and run a [single-host, multi-node testnet](20_local-multi-node-testnet.md). diff --git a/docs/01_nodeos/02_usage/03_development-environment/10_local-single-node-testnet-consensus.md b/docs/01_nodeos/02_usage/03_development-environment/10_local-single-node-testnet-consensus.md new file mode 100644 index 00000000000..8e5d4d92533 --- /dev/null +++ b/docs/01_nodeos/02_usage/03_development-environment/10_local-single-node-testnet-consensus.md @@ -0,0 +1,202 @@ +--- +content_title: Local Single-Node Testnet With Consensus Protocol +link_text: Local Single-Node Testnet With Consensus Protocol +--- + +## Goal + +This section describes how to set up a single-node blockchain configuration running on a single host with [consensus protocol](https://developers.eos.io/welcome/v2.1/protocol/consensus_protocol) enabled. This is referred to as a _**single host, single-node testnet with consensus**_. We will set up one node on your local computer and have it produce blocks. The following diagram depicts the desired single host testnet. + +![Single host single node testnet](single-host-single-node-testnet.png) + +`cleos` is used to interact with wallets, and send actions to the blockchain. `keosd` performs wallet management, including digital signing. If not started explicitly, `keosd` is started by `cleos` by default. + +## Before you begin + +* [Install the EOSIO software](../../../00_install/index.md) before starting this section. +* It is assumed that `nodeos`, `cleos`, and `keosd` are accessible through the path +* Know how to pass [Nodeos options](../../02_usage/00_nodeos-options.md) to enable or disable functionality. + +## Steps + +Open one "terminal" window and perform the following steps: + +1. [Add the development key to the wallet](#1-add-the-development-key-to-the-wallet) +2. [Start the Producer Node](#2-start-the-producer-node) +3. [Preactivate Protocol Features](#3-preactivate-protocol-features) +4. [Get the System Smart Contracts](#4-get-the-system-smart-contracts) +5. [Install eosio.boot System Contract](#5-install-eosioboot-system-contract) +6. [Activate the Remaining Protocol Features](#6-activate-the-remaining-protocol-features) +7. [Install eosio.bios System Contract](#7-install-eosiobios-system-contract) + +### 1. Add the development key to the wallet + +Execute the following at the command prompt: + +```sh +cleos wallet import --private-key 5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3 +``` + +### 2. Start the Producer Node + +Start your own single-node blockchain with this single command: + +```sh +nodeos -e -p eosio --plugin eosio::producer_plugin --plugin eosio::producer_api_plugin --plugin eosio::chain_api_plugin --access-control-allow-origin='*' --contracts-console --http-validate-host=false --verbose-http-errors +``` + +[[warning | Security Notice]] +| Do not use the parameters `--access-control-allow-origin='*'`, `--http-validate-host=false`, `--verbose-http-errors` and `--contracts-console`, in production because they either weaken the security or affect performance of your node. + +After running `nodeos`, you should get log messages similar to the ones below. It means the blocks are successfully produced. + +```sh +info 2021-01-07T15:59:07.902 thread-0 producer_plugin.cpp:2053 produce_block ] Produced block 98fa5cdd7ce06ae8... #162 @ 2021-01-07T15:59:08.000 signed by eosio [trxs: 0, lib: 161, confirmed: 0] +info 2021-01-07T15:59:08.401 thread-0 producer_plugin.cpp:2053 produce_block ] Produced block 972190051a840992... #163 @ 2021-01-07T15:59:08.500 signed by eosio [trxs: 0, lib: 162, confirmed: 0] +info 2021-01-07T15:59:08.901 thread-0 producer_plugin.cpp:2053 produce_block ] Produced block d8727439a26f36f6... #164 @ 2021-01-07T15:59:09.000 signed by eosio [trxs: 0, lib: 163, confirmed: 0] +info 2021-01-07T15:59:09.401 thread-0 producer_plugin.cpp:2053 produce_block ] Produced block 6f2302aeb944c5ca... #165 @ 2021-01-07T15:59:09.500 signed by eosio [trxs: 0, lib: 164, confirmed: 0] +... +info 2021-01-07T15:59:10.902 thread-0 producer_plugin.cpp:2053 produce_block ] Produced block 8cd2384a28818e19... #168 @ 2021-01-07T15:59:11.000 signed by eosio [trxs: 0, lib: 167, confirmed: 0] +... +info 2021-01-07T15:59:11.900 thread-0 producer_plugin.cpp:2053 produce_block ] Produced block c8c82b50249e9f6d... #170 @ 2021-01-07T15:59:12.000 signed by eosio [trxs: 0, lib: 169, confirmed: 0] +... +``` + +At this point, `nodeos` is running with a single producer, `eosio`. + +### 3. Preactivate Protocol Features + +All of the protocol upgrade features introduced in v1.8 and on subsequent versions require a special protocol feature, also known as `PREACTIVATE_FEATURE`, to be activated. + +To activate the special protocol `PREACTIVATE_FEATURE` run the following command from a terminal: + +```sh +curl --request POST \ + --url http://127.0.0.1:8888/v1/producer/schedule_protocol_feature_activations \ + -d '{"protocol_features_to_activate": ["0ec7e080177b2c02b278d5088611686b49d739925a92d9bfcacd7fc6b74053bd"]}' +``` + +### 4. Get the System Smart Contracts + +All of the protocol upgrade features introduced in v1.8 and on subsequent versions also require an updated version of the system smart contract which can make use of those protocol features. + +Two updated reference system smart contracts, `eosio.boot` and `eosio.bios`, are available in both source and binary form within the [`eos`](https://github.com/EOSIO/eos.git) repository. You can build them from source or deploy the binaries directly. + +#### 4.1 Use the Prebuilt System Smart Contracts + +To use the prebuilt system smart contract execute the following commands from a terminal: + +```sh +cd ~ +git clone https://github.com/EOSIO/eos.git +cd ./eos/contracts/contracts/ +pwd +``` + +Note the path printed at the command prompt, we will refer to it later as `EOSIO_SYSTEM_CONTRACTS_DIRECTORY`. + +Alternatively you can build the system smart contracts from source with the following commands: + +```sh +cd ~ +git clone https://github.com/EOSIO/eos.git +cd ./eos/contracts/contracts/ +mkdir build +cd build +cmake .. +make +pwd +``` + +### 5. Install eosio.boot System Contract + +To install the `eosio.boot` system contract execute the following command from a terminal. Make sure you replace the `EOSIO_SYSTEM_CONTRACTS_DIRECTORY` with the directory path where the `eosio.boot.wasm` and `eosio.boot.abi` files are located. + +```sh +cleos set contract eosio EOSIO_SYSTEM_CONTRACTS_DIRECTORY/eosio.boot/bin/ eosio.boot.wasm eosio.boot.abi +``` + +You should see something similar to the following output: + +```sh +Reading WAST/WASM from /users/documents/eos/contracts/contracts/eosio.boot/build/eosio.boot.wasm... +Using already assembled WASM... +Publishing contract... +executed transaction: 2150ed87e4564cd3fe98ccdea841dc9ff67351f9315b6384084e8572a35887cc 39968 bytes 4395 us +# eosio <= eosio::setcode {"account":"eosio","vmtype":0,"vmversion":0,"code":"0061736d0100000001be023060027f7e0060067f7e7e7f7f... +# eosio <= eosio::setabi {"account":"eosio","abi":{"types":[],"structs":[{"name":"buyrambytes","base":"","fields":[{"name":"p... +``` + +### 6. Activate the Remaining Protocol Features + +After you deploy the `eosio.boot` contract, run the following commands from a terminal to enable the rest of the features which are highly recommended to enable an EOSIO-based blockchain. + +[[info | Optional Step]] +|These features are optional. You can choose to enable or continue without these features; however they are highly recommended for an EOSIO-based blockchain. + +```sh +echo KV_DATABASE +cleos push action eosio activate '["825ee6288fb1373eab1b5187ec2f04f6eacb39cb3a97f356a07c91622dd61d16"]' -p eosio + +echo ACTION_RETURN_VALUE +cleos push action eosio activate '["c3a6138c5061cf291310887c0b5c71fcaffeab90d5deb50d3b9e687cead45071"]' -p eosio + +echo CONFIGURABLE_WASM_LIMITS +cleos push action eosio activate '["bf61537fd21c61a60e542a5d66c3f6a78da0589336868307f94a82bccea84e88"]' -p eosio + +echo BLOCKCHAIN_PARAMETERS +cleos push action eosio activate '["5443fcf88330c586bc0e5f3dee10e7f63c76c00249c87fe4fbf7f38c082006b4"]' -p eosio + +echo GET_SENDER +cleos push action eosio activate '["f0af56d2c5a48d60a4a5b5c903edfb7db3a736a94ed589d0b797df33ff9d3e1d"]' -p eosio + +echo FORWARD_SETCODE +cleos push action eosio activate '["2652f5f96006294109b3dd0bbde63693f55324af452b799ee137a81a905eed25"]' -p eosio + +echo ONLY_BILL_FIRST_AUTHORIZER +cleos push action eosio activate '["8ba52fe7a3956c5cd3a656a3174b931d3bb2abb45578befc59f283ecd816a405"]' -p eosio + +echo RESTRICT_ACTION_TO_SELF +cleos push action eosio activate '["ad9e3d8f650687709fd68f4b90b41f7d825a365b02c23a636cef88ac2ac00c43"]' -p eosio + +echo DISALLOW_EMPTY_PRODUCER_SCHEDULE +cleos push action eosio activate '["68dcaa34c0517d19666e6b33add67351d8c5f69e999ca1e37931bc410a297428"]' -p eosio + + echo FIX_LINKAUTH_RESTRICTION +cleos push action eosio activate '["e0fb64b1085cc5538970158d05a009c24e276fb94e1a0bf6a528b48fbc4ff526"]' -p eosio + + echo REPLACE_DEFERRED +cleos push action eosio activate '["ef43112c6543b88db2283a2e077278c315ae2c84719a8b25f25cc88565fbea99"]' -p eosio + +echo NO_DUPLICATE_DEFERRED_ID +cleos push action eosio activate '["4a90c00d55454dc5b059055ca213579c6ea856967712a56017487886a4d4cc0f"]' -p eosio + +echo ONLY_LINK_TO_EXISTING_PERMISSION +cleos push action eosio activate '["1a99a59d87e06e09ec5b028a9cbb7749b4a5ad8819004365d02dc4379a8b7241"]' -p eosio + +echo RAM_RESTRICTIONS +cleos push action eosio activate '["4e7bf348da00a945489b2a681749eb56f5de00b900014e137ddae39f48f69d67"]' -p eosio + +echo WEBAUTHN_KEY +cleos push action eosio activate '["4fca8bd82bbd181e714e283f83e1b45d95ca5af40fb89ad3977b653c448f78c2"]' -p eosio + +echo WTMSIG_BLOCK_SIGNATURES +cleos push action eosio activate '["299dcb6af692324b899b39f16d5a530a33062804e41f09dc97e9f156b4476707"]' -p eosio +``` + +### 7. Install eosio.bios System Contract + +To deploy the `eosio.bios` system contract execute the following command from a terminal. Make sure you replace the `EOSIO_SYSTEM_CONTRACTS_DIRECTORY` with the directory path where the `eosio.bios.wasm` and `eosio.bios.abi` files are located. + +```sh +cleos set contract eosio EOSIO_SYSTEM_CONTRACTS_DIRECTORY/eosio.bios/bin/ eosio.bios.wasm eosio.bios.abi +``` + +If you want instead of `eosio.bios`, a more elaborate system smart contract like `eosio.system`, you have to build it first and then execute the following command from a terminal (or from the shell). Make sure you replace the `EOSIO_SYSTEM_CONTRACTS_DIRECTORY` with the directory path where you have built the `eosio.system` system contract, the directory path where `eoios.system.abi` and `eosio.system.wasm` are located. + +```sh +cleos set contract eosio EOSIO_SYSTEM_CONTRACTS_DIRECTORY/eosio.system/ eosio.system.wasm eosio.system.abi +``` + +[[info | What's next?]] +| We will explore how to setup and run a [single-host, multi-node testnet](20_local-multi-node-testnet.md). diff --git a/docs/01_nodeos/02_usage/03_development-environment/01_local-multi-node-testnet.md b/docs/01_nodeos/02_usage/03_development-environment/20_local-multi-node-testnet.md similarity index 100% rename from docs/01_nodeos/02_usage/03_development-environment/01_local-multi-node-testnet.md rename to docs/01_nodeos/02_usage/03_development-environment/20_local-multi-node-testnet.md diff --git a/docs/01_nodeos/02_usage/03_development-environment/index.md b/docs/01_nodeos/02_usage/03_development-environment/index.md index f7cfa8f8824..9b099902e01 100644 --- a/docs/01_nodeos/02_usage/03_development-environment/index.md +++ b/docs/01_nodeos/02_usage/03_development-environment/index.md @@ -10,11 +10,17 @@ This is the go-to option for smart contract developers, aspiring Block Producers * [Configure Nodeos as a Local Single-node Testnet](00_local-single-node-testnet.md) +## Local Single-Node Testnet With Consensus Protocol + +This is the go-to option for smart contract developers, aspiring Block Producers or Non-Producing Node operators. It has the most simple configuration with the least number of requirements to setup a local single-node testnet with consensus protocol enabled. + +* [Configure Nodeos as a Local Single-node Testnet with Consensus Protocol](10_local-single-node-testnet-consensus.md) + ## Local Multi-Node Testnet While this option can technically be used for smart contract development, it may be overkill. This is most beneficial for those who are working on aspects of core development, such as benchmarking, optimization and experimentation. It's also a good option for hands-on learning and concept proofing. -* [Configure Nodeos as a Local Two-Node Testnet](01_local-multi-node-testnet.md) +* [Configure Nodeos as a Local Two-Node Testnet](20_local-multi-node-testnet.md) * [Configure Nodeos as a Local 21-Node Testnet](https://github.com/EOSIO/eos/blob/master/tutorials/bios-boot-tutorial/README.md) ## Official Testnet diff --git a/docs/01_nodeos/02_usage/60_how-to-guides/10_how-to-configure-state-storage.md b/docs/01_nodeos/02_usage/60_how-to-guides/10_how-to-configure-state-storage.md new file mode 100644 index 00000000000..656c2e1a8ba --- /dev/null +++ b/docs/01_nodeos/02_usage/60_how-to-guides/10_how-to-configure-state-storage.md @@ -0,0 +1,45 @@ +# Summary +This how-to describes configuration of the Nodeos `backing store`. `Nodeos` can now use `chainbase` or `rocksdb` as a backing store for smart contract state. + +# Prerequisites +Version 2.1 or above of the EOSIO development environment. + +# Parameter Definitions +Specify which backing store to use with the `chain_plugin` `--backing-store` argument. This argument sets state storage to either `chainbase`, the default, or `rocksdb`. + +```console +Config Options for eosio::chain_plugin: + + --backing-store arg (=chainbase) The storage for state, chainbase or + rocksdb + --persistent-storage-num-threads arg Number of rocksdb threads for flush and + compaction. + --persistent-storage-max-num-files arg (=-1) + Max number of rocksdb files to keep + open. -1 = unlimited. + --persistent-storage-write-buffer-size-mb arg (=128) + Size of a single rocksdb memtable (in MiB). + --persistent-storage-bytes-per-sync Rocksdb write rate of flushes and compactions. + --persistent-storage-mbytes-snapshot-batch + Rocksdb batch size threshold before writing read in snapshot data to database. +``` + +# Procedure +To use `rocksdb` for state storage: + +```shell +nodeos -e -p eosio --plugin eosio::producer_plugin --plugin eosio::producer_api_plugin --plugin eosio::chain_api_plugin --backing-store=’rocksdb’ --persistent-storage-num-threads=’2’ --persistent-storage-max-num-files=’2’ --persistent-storage-write-buffer-size-mb=’128’ --plugin eosio::http_plugin +``` + +To use `chainbase` for state storage: + +```shell +nodeos -e -p eosio --plugin eosio::producer_plugin --plugin eosio::producer_api_plugin --plugin eosio::chain_api_plugin --backing-store=’chainbase’ --plugin eosio::http_plugin +``` + +or + +```shell +nodeos -e -p eosio --plugin eosio::producer_plugin --plugin eosio::producer_api_plugin --plugin eosio::chain_api_plugin --plugin eosio::http_plugin +``` + diff --git a/docs/01_nodeos/02_usage/index.md b/docs/01_nodeos/02_usage/index.md index 2ab1a5a7ae1..2d9b47842cb 100644 --- a/docs/01_nodeos/02_usage/index.md +++ b/docs/01_nodeos/02_usage/index.md @@ -8,3 +8,4 @@ This section explains how to use `nodeos`, lists its configuration options, desc * [Configuration](01_nodeos-configuration.md) - CLI vs. `config.ini` options; `nodeos` example. * [Node Setups](02_node-setups/index.md) - Producing vs. non-producing nodes setup. * [Development Environment](03_development-environment/index.md) - Setting up a development/test environment. +* [How to Configure State Storage](60_how-to-guides/10_how-to-configure-state-storage.md) - How to configure the `nodeos` `backing store`. diff --git a/docs/01_nodeos/03_plugins/blockvault_client_plugin/index.md b/docs/01_nodeos/03_plugins/blockvault_client_plugin/index.md new file mode 100644 index 00000000000..0dabc023b6e --- /dev/null +++ b/docs/01_nodeos/03_plugins/blockvault_client_plugin/index.md @@ -0,0 +1,114 @@ + +## Overview + +The `blockvault_client_plugin` enables blockchain administrators to implement industry standard disaster recovery to maximize producer operational uptime. The plugin allows a block producer to cluster two or more nodes deployed as a single logical producer. If one of the nodes goes down, the other nodes in the cluster continue to operate, thereby meeting certain guarantees for the producer to continue to function with minimal service disruption. + +## Goals + +Block Vault is a clustered component within an EOSIO network architecture that enables replicated durable storage with strong consistency guarantees for the input required by a redundant cluster of nodes. In particular, Block Vault achieves the following guarantees for any cluster node running `nodeos` configured as a Block Vault client in producing mode: + +* Guarantee against double-production of blocks +* Guarantee against finality violation +* Guarantee of liveness (ability to make progress as a blockchain) + +To facilitate these guarantees, Block Vault allows `nodeos` to run in a redundant and/or highly available mode. Block Vault itself does not implement any coordination of nodes in a cluster. It merely guarantees that any such coordination, including faulty coordination leading to multiple active block constructing nodes, will be safe as defined by the above guarantees. For more information, read the [Block Vault Operation](#block-vault-operation) section below. + +## Usage + +```console +# config.ini +plugin = eosio::blockvault_client_plugin +[options] +``` +```sh +# command-line +nodeos ... --plugin eosio::blockvault_client_plugin [options] +``` + +## Configuration Options + +These can be specified from both the `nodeos` command-line or the `config.ini` file: + +```console +Config Options for eosio::blockvault_client_plugin: + + --block-vault-backend arg the uri for block vault backend. + Currently, only PostgreSQL is + supported, the format is + 'postgresql://username:password@localho + st/company' +``` + +## Plugin Dependencies + +* [`producer_plugin`](../producer_plugin/index.md) + +## Configuration Example + +To use `blockvault_client_plugin`, the `nodeos` service must be configured as a producer with the `--block-vault-backend` option: + +```sh +nodeos --plugin eosio::producer_plugin --producer-name myproducer --plugin eosio::blockvault_client_plugin --block-vault-backend postgresql://user:password@mycompany.com +``` + +For production deployments, it is recommend to use the `PGPASSWORD` environment variable to configure the password, instead of embedding the password in the URI. + +```sh +export PGPASSWORD=password +nodeos --plugin eosio::producer_plugin --producer-name myproducer --plugin eosio::blockvault_client_plugin --block-vault-backend postgresql://user@mycompany.com +``` + +## Software Dependencies + +To build `blockvault_client_plugin` you need `libpq` version 10 or above and `libpqxx` version 6 or above. These dependencies are typically installed (alongside other dependencies) when you either [Install EOSIO](../../../00_install/index.md) from prebuilt binaries or build from source. You may also opt to install these dependencies manually prior to installing or building EOSIO. + +For MacOS, you can simply use homebrew to install these dependencies: + +```sh +brew install libpq libpqxx +``` + +For Linux, the versions of `libpq` and `libpqxx` provided by the system package managers may not be current enough. We recommend to follow the `install libpq` and `install libpqxx` sections of the corresponding dockerfile in `.cicd/platform/pinned` for your platform to install these dependencies. + +## Block Vault Storage + +Currently, Block Vault uses `PostgreSQL` for its durable storage. Other distributed databases may be supported in the future. + +### Running PostgreSQL for Testing + +We recommend to use `docker`: + +```sh +docker run -p 5432:5432 -e POSTGRES_PASSWORD=password -d postgres +``` + +### Running PostgreSQL for Production + +We recommend to deploy `PostgreSQL` with HA (high availability) mode and synchronous replication strategy. + +### Database Schema + +`blockvault_client_plugin` creates two tables `BlockData` and `SnapshotData` if not already in the database. The tables are created with the following SQL commands when the plugin starts: + +```sh +CREATE TABLE IF NOT EXISTS BlockData (watermark_bn bigint, watermark_ts bigint, lib bigint, block_num bigint, block_id bytea UNIQUE, previous_block_id bytea, block oid, block_size bigint); + +CREATE TABLE IF NOT EXISTS SnapshotData (watermark_bn bigint, watermark_ts bigint, snapshot oid); +``` + +## Block Vault Operation + +As new nodes join a cluster, the Block Vault will be their exclusive source of sync data, enabling it to guarantee a consistent view of the blockchain as a base. When a node participating in the cluster constructs a new block, it will submit it to the Block Vault for approval prior to broadcasting it to external peers via the P2P network. + +Block Vault is exclusively responsible for providing guarantees against double-production of blocks and finality violations. It facilitates partial guarantee of liveness through node redundancy by ensuring that data needed to construct new blocks are durable and replicated. However, Block Vault cannot guarantee that the data it contains was not malformed. To that end, Block Vault depends on the proper operation of the clustered nodes. + +## Block Vault Client API + +Cluster nodes interact with the Block Vault through the following messages: + +* [`async_propose_constructed_block()`](../../../classeosio_1_1blockvault_1_1block__vault__interface#function-async_propose_constructed_block) +* [`async_append_external_block()`](../../../classeosio_1_1blockvault_1_1block__vault__interface#function-async_append_external_block) +* [`propose_snapshot()`](../../../classeosio_1_1blockvault_1_1block__vault__interface#function-propose_snapshot) +* [`sync()`](../../../classeosio_1_1blockvault_1_1block__vault__interface#function-sync) + +For more information visit the [block_vault_interface](../../../classeosio_1_1blockvault_1_1block__vault__interface) C++ reference. diff --git a/docs/01_nodeos/03_plugins/chain_plugin/index.md b/docs/01_nodeos/03_plugins/chain_plugin/index.md index 47304291e8f..66d43b35603 100644 --- a/docs/01_nodeos/03_plugins/chain_plugin/index.md +++ b/docs/01_nodeos/03_plugins/chain_plugin/index.md @@ -29,11 +29,16 @@ Command Line Options for eosio::chain_plugin: --extract-genesis-json arg extract genesis_state from blocks.log as JSON, write into specified file, and exit + --print-build-info print build environment information to + console as JSON and exit + --extract-build-info arg extract build environment information + as JSON, write into specified file, and + exit --fix-reversible-blocks recovers reversible block database if that database is in a bad state - --force-all-checks do not skip any checks that can be - skipped while replaying irreversible - blocks + --force-all-checks do not skip any validation checks while + replaying blocks (useful for replaying + blocks from untrusted source) --disable-replay-opts disable optimizations that specifically target replay --replay-blockchain clear chain state database and replay @@ -46,6 +51,8 @@ Command Line Options for eosio::chain_plugin: --truncate-at-block arg (=0) stop hard replay / block log recovery at this block number (if set to non-zero number) + --terminate-at-block arg (=0) terminate after reaching this block + number (if set to a non-zero number) --import-reversible-blocks arg replace reversible block database with blocks imported from specified file and then exit @@ -65,15 +72,63 @@ Config Options for eosio::chain_plugin: --blocks-dir arg (="blocks") the location of the blocks directory (absolute path or relative to application data dir) + --blocks-log-stride arg (=4294967295) split the block log file when the head + block number is the multiple of the + stride + When the stride is reached, the current + block log and index will be renamed + '/blocks--.log/index' + and a new current block log and index + will be created with the most recent + block. All files following + this format will be used to construct + an extended block log. + --max-retained-block-files arg (=10) the maximum number of blocks files to + retain so that the blocks in those + files can be queried. + When the number is reached, the oldest + block file would be moved to archive + dir or deleted if the archive dir is + empty. + The retained block log files should not + be manipulated by users. + --blocks-retained-dir arg (="") the location of the blocks retained + directory (absolute path or relative to + blocks dir). + If the value is empty, it is set to the + value of blocks dir. + --blocks-archive-dir arg (="archive") the location of the blocks archive + directory (absolute path or relative to + blocks dir). + If the value is empty, blocks files + beyond the retained limit will be + deleted. + All files in the archive directory are + completely under user's control, i.e. + they won't be accessed by nodeos + anymore. + --fix-irreversible-blocks arg (=1) When the existing block log is + inconsistent with the index, allows + fixing the block log and index files + automatically - that is, it will take + the highest indexed block if it is + valid; otherwise it will repair the + block log and reconstruct the index. --protocol-features-dir arg (="protocol_features") the location of the protocol_features directory (absolute path or relative to application config dir) --checkpoint arg Pairs of [BLOCK_NUM,BLOCK_ID] that should be enforced as checkpoints. - --wasm-runtime eos-vm|eos-vm-jit Override default WASM runtime (wabt) - --eos-vm-oc-enable Enable optimized compilation in WASM - --abi-serializer-max-time-ms arg (=15000) + --wasm-runtime runtime (=eos-vm-jit) Override default WASM runtime ( + "eos-vm-jit", "eos-vm") + "eos-vm-jit" : A WebAssembly runtime + that compiles WebAssembly code to + native x86 code prior to execution. + "eos-vm" : A WebAssembly interpreter. + + --abi-serializer-max-time-ms arg (=15) Override default maximum ABI serialization time allowed in ms --chain-state-db-size-mb arg (=1024) Maximum size (in MiB) of the chain @@ -82,6 +137,24 @@ Config Options for eosio::chain_plugin: Safely shut down node when free space remaining in the chain state database drops below this size (in MiB). + --backing-store arg (=chainbase) The storage for state, chainbase or + rocksdb + --persistent-storage-num-threads arg (=1) + Number of rocksdb threads for flush and + compaction + --persistent-storage-max-num-files arg (=-1) + Max number of rocksdb files to keep + open. -1 = unlimited. + --persistent-storage-write-buffer-size-mb arg (=128) + Size of a single rocksdb memtable (in + MiB) + --persistent-storage-bytes-per-sync arg (=1048576) + Rocksdb write rate of flushes and + compactions. + --persistent-storage-mbytes-snapshot-batch arg (=50) + Rocksdb batch size threshold before + writing read in snapshot data to + database. --reversible-blocks-db-size-mb arg (=340) Maximum size (in MiB) of the reversible blocks database @@ -97,6 +170,14 @@ Config Options for eosio::chain_plugin: --chain-threads arg (=2) Number of worker threads in controller thread pool --contracts-console print contract's output to console + --deep-mind print deeper information about chain + operations + --telemetry-url arg Send Zipkin spans to url. e.g. + http://127.0.0.1:9411/api/v2/spans + --telemetry-service-name arg (=nodeos) + Zipkin localEndpoint.serviceName sent + with each span + --telemetry-timeout-us arg (=200000) Timeout for sending Zipkin span. --actor-whitelist arg Account added to actor whitelist (may specify multiple times) --actor-blacklist arg Account added to actor blacklist (may @@ -119,22 +200,36 @@ Config Options for eosio::chain_plugin: times) --read-mode arg (=speculative) Database read mode ("speculative", "head", "read-only", "irreversible"). - In "speculative" mode database contains - changes done up to the head block plus - changes made by transactions not yet - included to the blockchain. - In "head" mode database contains - changes done up to the current head - block. - In "read-only" mode database contains - changes done up to the current head - block and transactions cannot be pushed - to the chain API. - In "irreversible" mode database - contains changes done up to the last - irreversible block and transactions - cannot be pushed to the chain API. + In "speculative" mode: database + contains state changes by transactions + in the blockchain up to the head block + as well as some transactions not yet + included in the blockchain. + In "head" mode: database contains state + changes by only transactions in the + blockchain up to the head block; + transactions received by the node are + relayed if valid. + In "read-only" mode: (DEPRECATED: see + p2p-accept-transactions & + api-accept-transactions) database + contains state changes by only + transactions in the blockchain up to + the head block; transactions received + via the P2P network are not relayed and + transactions cannot be pushed via the + chain API. + In "irreversible" mode: database + contains state changes by only + transactions in the blockchain up to + the last irreversible block; + transactions received via the P2P + network are not relayed and + transactions cannot be pushed via the + chain API. + --api-accept-transactions arg (=1) Allow API transactions to be evaluated + and relayed if valid. --validation-mode arg (=full) Chain validation mode ("full" or "light"). In "full" mode all incoming blocks will @@ -150,6 +245,10 @@ Config Options for eosio::chain_plugin: context of a notification handler (i.e. when the receiver is not the code of the action). + --maximum-variable-signature-length arg (=16384) + Subjectively limit the maximum length + of variable components in a variable + legnth signature to this size in bytes --trusted-producer arg Indicate a producer whose blocks headers signed by it will be fully validated, but transactions in those @@ -159,21 +258,18 @@ Config Options for eosio::chain_plugin: In "mapped" mode database is memory mapped as a file. In "heap" mode database is preloaded in - to swappable memory. + to swappable memory and will use huge + pages if available. In "locked" mode database is preloaded, - locked in to memory, and optionally can - use huge pages. - - --database-hugepage-path arg Optional path for database hugepages - when in "locked" mode (may specify - multiple times) + locked in to memory, and will use huge + pages if available. - --max-nonprivileged-inline-action-size arg - Sets the maximum limit for - non-privileged inline actions. - The default value is 4 KB - and if this threshold is exceeded, - the transaction will subjectively fail. + --enable-account-queries arg (=0) enable queries to find accounts by + various metadata. + --max-nonprivileged-inline-action-size arg (=4096) + maximum allowed size (in bytes) of an + inline action for a nonprivileged + account ``` ## Dependencies diff --git a/docs/01_nodeos/03_plugins/faucet_testnet_plugin/index.md b/docs/01_nodeos/03_plugins/faucet_testnet_plugin/index.md deleted file mode 100644 index 46c47af5755..00000000000 --- a/docs/01_nodeos/03_plugins/faucet_testnet_plugin/index.md +++ /dev/null @@ -1,22 +0,0 @@ -## Description - -The `faucet_testnet_plugin` provides an interface that assists in the automation of distributing tokens on an EOSIO testnet. - -## Usage - -```console -# config.ini -plugin = eosio::faucet_testnet_plugin -``` -```sh -# command-line -nodeos ... --plugin eosio::faucet_testnet_plugin -``` - -## Options - -None - -## Dependencies - -* [`http_plugin`](../http_plugin/index.md) diff --git a/docs/01_nodeos/03_plugins/history_api_plugin/index.md b/docs/01_nodeos/03_plugins/history_api_plugin/index.md index df8e448aca2..c64319432d4 100644 --- a/docs/01_nodeos/03_plugins/history_api_plugin/index.md +++ b/docs/01_nodeos/03_plugins/history_api_plugin/index.md @@ -1,5 +1,5 @@ [[warning | Deprecation Notice]] -| The `history_plugin` that the `history_api_plugin` depends upon is deprecated and will no longer be maintained. Please use the [`state_history_plugin`](../state_history_plugin/index.md) instead. +| The `history_plugin` that the `history_api_plugin` depends upon is deprecated and will no longer be maintained. Please use the [`state_history_plugin`](../state_history_plugin/index.md) or the [`trace_api_plugin`](../trace_api_plugin/index.md) instead. ## Description diff --git a/docs/01_nodeos/03_plugins/history_plugin/index.md b/docs/01_nodeos/03_plugins/history_plugin/index.md index 0a39679c298..008571c270c 100644 --- a/docs/01_nodeos/03_plugins/history_plugin/index.md +++ b/docs/01_nodeos/03_plugins/history_plugin/index.md @@ -1,5 +1,5 @@ [[warning | Deprecation Notice]] -| The `history_plugin` is deprecated and will no longer be maintained. Please use the [`state_history_plugin`](../state_history_plugin/index.md) instead. +| The `history_plugin` is deprecated and will no longer be maintained. Please use the [`state_history_plugin`](../state_history_plugin/index.md) or the [`trace_api_plugin`](../trace_api_plugin/index.md) instead. ## Description diff --git a/docs/01_nodeos/03_plugins/http_plugin/index.md b/docs/01_nodeos/03_plugins/http_plugin/index.md index 17f5fb88530..9f58a603dae 100644 --- a/docs/01_nodeos/03_plugins/http_plugin/index.md +++ b/docs/01_nodeos/03_plugins/http_plugin/index.md @@ -22,10 +22,11 @@ These can be specified from both the command-line or the `config.ini` file: ```console Config Options for eosio::http_plugin: + --unix-socket-path arg The filename (relative to data-dir) to create a unix socket for HTTP RPC; set - blank to disable (=keosd.sock for keosd) - --http-server-address arg (=127.0.0.1:8888 for nodeos) + blank to disable. + --http-server-address arg (=127.0.0.1:8888) The local IP and port to listen for incoming http connections; set blank to disable. @@ -53,8 +54,14 @@ Config Options for eosio::http_plugin: --http-max-bytes-in-flight-mb arg (=500) Maximum size in megabytes http_plugin should use for processing http - requests. 503 error response when + requests. 429 error response when + exceeded. + --http-max-in-flight-requests arg (=-1) + Maximum number of requests http_plugin + should use for processing http + requests. 429 error response when exceeded. + --http-max-response-time-ms arg (=30) Maximum time for processing a request. --verbose-http-errors Append the error log to HTTP responses --http-validate-host arg (=1) If set to false, then any incoming "Host" header is considered valid diff --git a/docs/01_nodeos/03_plugins/index.md b/docs/01_nodeos/03_plugins/index.md index fd05571afbf..5c9aac2ca42 100644 --- a/docs/01_nodeos/03_plugins/index.md +++ b/docs/01_nodeos/03_plugins/index.md @@ -8,10 +8,10 @@ Plugins extend the core functionality implemented in `nodeos`. Some plugins are For information on specific plugins, just select from the list below: +* [`blockvault_client_plugin`](blockvault_client_plugin/index.md) * [`chain_api_plugin`](chain_api_plugin/index.md) * [`chain_plugin`](chain_plugin/index.md) * [`db_size_api_plugin`](db_size_api_plugin/index.md) -* [`faucet_testnet_plugin`](faucet_testnet_plugin/index.md) * [`history_api_plugin`](history_api_plugin/index.md) * [`history_plugin`](history_plugin/index.md) * [`http_client_plugin`](http_client_plugin/index.md) @@ -21,8 +21,6 @@ For information on specific plugins, just select from the list below: * [`net_plugin`](net_plugin/index.md) * [`producer_plugin`](producer_plugin/index.md) * [`state_history_plugin`](state_history_plugin/index.md) -* [`test_control_api_plugin`](test_control_api_plugin/index.md) -* [`test_control_plugin`](test_control_plugin/index.md) * [`trace_api_plugin`](trace_api_plugin/index.md) * [`txn_test_gen_plugin`](txn_test_gen_plugin/index.md) diff --git a/docs/01_nodeos/03_plugins/mongo_db_plugin/index.md b/docs/01_nodeos/03_plugins/mongo_db_plugin/index.md deleted file mode 100644 index a750fe89b59..00000000000 --- a/docs/01_nodeos/03_plugins/mongo_db_plugin/index.md +++ /dev/null @@ -1,153 +0,0 @@ -[[warning | Deprecation Notice]] -| The `mongo_db_plugin` is deprecated and will no longer be maintained. Please refer to the [`state_history_plugin`](../state_history_plugin/index.md) and the [`history-tools`](../state_history_plugin/index.md#history-tools) for better options to archive blockchain data. - -## Description - -The optional `eosio::mongo_db_plugin` provides archiving of blockchain data into a MongoDB. It is recommended that the plugin be added to a non-producing node as it is designed to shut down on any failed insert into the MongoDB and it is resource intensive. For best results dedicate a `nodeos` instance to running this one plugin. The rationale behind this shutdown on error is so that any issues with connectivity or the mongo database can be fixed and `nodeos` can be restarted without having to resync or replay. - -## Important Notes - -* Documents stored in mongo by `mongo_db_plugin` which contain empty field/struct names will be stored with the field/struct name of `empty_field_name` / `empty_struct_name`. -* Action data is stored on chain as raw bytes. This plugin attempts to use associated ABI on accounts to deserialize the raw bytes into expanded `abi_def` form for storage into mongo. Note that invalid or missing ABI on a contract will result in the action data being stored as raw bytes. For example the EOSIO system contract does not provide ABI for the `onblock` action so it is stored as raw bytes. -* The `mongo_db_plugin` does slow down replay/resync as the conversion of block data to JSON and insertion into MongoDB is resource intensive. The plugin does use a worker thread for processing the block data, but this does not help much when replaying/resyncing. - -## Recommendations - -* It is recommended that a large `--abi-serializer-max-time-ms` value be passed into the `nodeos` running the `mongo_db_plugin` as the default ABI serializer time limit is not large enough to serialize large blocks. -* Read-only mode should be used to avoid speculative execution. See [Nodeos Read Modes](../../02_usage/05_nodeos-implementation.md#nodeos-read-modes). Forked data is still recorded (data that never becomes irreversible) but speculative transaction processing and signaling is avoided, minimizing the transaction_traces/action_traces stored. - -## Options - -These can be specified from both the command-line or the `config.ini` file: - -```console - -q [ --mongodb-queue-size ] arg (=256) - The target queue size between nodeos - and MongoDB plugin thread. - --mongodb-abi-cache-size The maximum size of the abi cache for - serializing data. - --mongodb-wipe Required with --replay-blockchain, - --hard-replay-blockchain, or - --delete-all-blocks to wipe mongo - db.This option required to prevent - accidental wipe of mongo db. - Defaults to false. - --mongodb-block-start arg (=0) If specified then only abi data pushed - to mongodb until specified block is - reached. - -m [ --mongodb-uri ] arg MongoDB URI connection string, see: - https://docs.mongodb.com/master/referen - ce/connection-string/. If not specified - then plugin is disabled. Default - database 'EOS' is used if not specified - in URI. Example: mongodb://127.0.0.1:27 - 017/EOS - --mongodb-update-via-block-num arg (=0) - Update blocks/block_state with latest - via block number so that duplicates are - overwritten. - --mongodb-store-blocks Enables storing blocks in mongodb. - Defaults to true. - --mongodb-store-block-states Enables storing block state in mongodb. - Defaults to true. - --mongodb-store-transactions Enables storing transactions in mongodb. - Defaults to true. - --mongodb-store-transaction-traces Enables storing transaction traces in mongodb. - Defaults to true. - --mongodb-store-action-traces Enables storing action traces in mongodb. - Defaults to true. - --mongodb-filter-on Mongodb: Track actions which match - receiver:action:actor. Actor may be blank - to include all. Receiver and Action may - not be blank. Default is * include - everything. - --mongodb-filter-out Mongodb: Do not track actions which match - receiver:action:actor. Action and Actor - both blank excludes all from reciever. Actor blank excludes all from - reciever:action. Receiver may not be - blank. -``` - -## Notes - -* `--mongodb-store-*` options all default to true. -* `--mongodb-filter-*` options currently only applies to the `action_traces` collection. - -## Example Filters - -```console -mongodb-filter-out = eosio:onblock: -mongodb-filter-out = gu2tembqgage:: -mongodb-filter-out = blocktwitter:: -``` - -[[warning | Warning]] -| When the `mongo_db_plugin` is turned on, the target mongodb instance may take a lot of storage space. With all collections enabled and no filters applied, the mongodb data folder can easily occupy hundreds of GBs of data. It is recommended that you tailor the options and utilize the filters as you need in order to maximize storage efficiency. - -## Collections - -* `accounts` - created on applied transaction. Always updated even if `mongodb-store-action-traces=false`. - * Currently limited to just name and ABI if contract abi on account - * Mostly for internal use as the stored ABI is used to convert action data into JSON for storage as associated actions on contract are processed. - * Invalid ABI on account will prevent conversion of action data into JSON for storage resulting in just the action data being stored as hex. For example, the original eosio.system contract did not provide ABI for the `onblock` action and therefore all `onblock` action data is stored as hex until the time `onblock` ABI is added to the eosio.system contract. - -* `action_traces` - created on applied transaction - * `receipt` - action_trace action_receipt - see `eosio::chain::action_receipt` - * `trx_id` - transaction id - * `act` - action - see `eosio::chain::action` - * `elapsed` - time in microseconds to execute action - * `console` - console output of action. Always empty unless `contracts-console = true` option specified. - -* `block_states` - created on accepted block - * `block_num` - * `block_id` - * `block_header_state` - see `eosio::chain::block_header_state` - * `validated` - * `in_current_chain` - -* `blocks` - created on accepted block - * `block_num` - * `block_id` - * `block` - signed block - see `eosio::chain::signed_block` - * `validated` - added on irreversible block - * `in_current_chain` - added on irreversible block - * `irreversible=true` - added on irreversible block - -* `transaction_traces` - created on applied transaction - * see `chain::eosio::transaction_trace` - -* `transactions` - created on accepted transaction - does not include inline actions - * see `eosio::chain::signed_transaction`. In addition to signed_transaction data the following are also stored. - * `trx_id` - transaction id - * `irreversible=true` - added on irreversible block - * `block_id` - added on irreversble block - * `block_num` - added on irreversible block - * `signing_keys` - * `accepted` - * `implicit` - * `scheduled` - -* `account_controls` - created on applied transaction. Always updated even if `mongodb-store-action-traces=false`. - * `controlled_account` - * `controlling_permission` - * `controlling_account` - -The equivalent of `/v1/history/get_controlled_acounts` with mongo: `db.account_controls.find({"controlling_account":"hellozhangyi"}).pretty()` - -* `pub_keys` - created on applied transaction. Always updated even if `mongodb-store-action-traces=false`. - * `account` - * `permission` - * `public_key` - -## Examples - -The mongodb equivalent of `/v1/history/get_key_accounts` RPC API endpoint: - -```console -db.pub_keys.find({"public_key":"EOS7EarnUhcyYqmdnPon8rm7mBCTnBoot6o7fE2WzjvEX2TdggbL3"}).pretty() -``` - -## Dependencies - -* [`chain_plugin`](../chain_plugin/index.md) -* [`history_plugin`](../history_plugin/index.md) diff --git a/docs/01_nodeos/03_plugins/net_api_plugin/index.md b/docs/01_nodeos/03_plugins/net_api_plugin/index.md index 6efefefb2b7..ac7ca7273fd 100644 --- a/docs/01_nodeos/03_plugins/net_api_plugin/index.md +++ b/docs/01_nodeos/03_plugins/net_api_plugin/index.md @@ -1,6 +1,5 @@ ## Description - -The `net_api_plugin` exposes functionality from the `net_plugin` to the RPC API interface managed by the `http_plugin`. +The `net_api_plugin` exposes functionality from the `net_plugin` to the RPC API interface managed by the `http_plugin`. Node operators can use the `net_api_plugin` to manage the p2p connections of an active node. The `net_api_plugin` provides four RPC API endpoints: diff --git a/docs/01_nodeos/03_plugins/net_plugin/index.md b/docs/01_nodeos/03_plugins/net_plugin/index.md index 963c8e73478..1d03aeb0bf4 100644 --- a/docs/01_nodeos/03_plugins/net_plugin/index.md +++ b/docs/01_nodeos/03_plugins/net_plugin/index.md @@ -31,9 +31,24 @@ Config Options for eosio::net_plugin: connect to. Use multiple p2p-peer-address options as needed to compose a network. + Syntax: host:port[:|] + The optional 'trx' and 'blk' + indicates to node that only + transactions 'trx' or blocks 'blk' + should be sent. Examples: + p2p.eos.io:9876 + p2p.trx.eos.io:9876:trx + p2p.blk.eos.io:9876:blk + --p2p-max-nodes-per-host arg (=1) Maximum number of client nodes from any single IP address - --agent-name arg (="EOS Test Agent") The name supplied to identify this node + --p2p-accept-transactions arg (=1) Allow transactions received over p2p + network to be evaluated and relayed if + valid. + --p2p-reject-incomplete-blocks arg (=1) + Reject pruned signed_blocks even in + light validation + --agent-name arg (=EOS Test Agent) The name supplied to identify this node amongst the peers. --allowed-connection arg (=any) Can be 'any' or 'producers' or 'specified' or 'none'. If 'specified', @@ -52,14 +67,12 @@ Config Options for eosio::net_plugin: cleaning up dead connections --max-cleanup-time-msec arg (=10) max connection cleanup time per cleanup call in millisec - --network-version-match arg (=0) True to require exact match of peer - network version. - --net-threads arg (=1) Number of worker threads in net_plugin + --net-threads arg (=2) Number of worker threads in net_plugin thread pool --sync-fetch-span arg (=100) number of blocks to retrieve in a chunk from any individual peer during synchronization - --use-socket-read-watermark arg (=0) Enable expirimental socket read + --use-socket-read-watermark arg (=0) Enable experimental socket read watermark optimization --peer-log-format arg (=["${_name}" ${_ip}:${_port}]) The string used to format peers when @@ -82,7 +95,10 @@ Config Options for eosio::net_plugin: peer _lport local port number connected - to peer + to peer + --p2p-keepalive-interval-ms arg (=32000) + peer heartbeat keepalive message + interval in milliseconds ``` ## Dependencies diff --git a/docs/01_nodeos/03_plugins/producer_plugin/index.md b/docs/01_nodeos/03_plugins/producer_plugin/index.md index 07f38ac8e76..5295a8b50dc 100644 --- a/docs/01_nodeos/03_plugins/producer_plugin/index.md +++ b/docs/01_nodeos/03_plugins/producer_plugin/index.md @@ -37,14 +37,6 @@ Config Options for eosio::producer_plugin: the DPOS Irreversible Block for a chain this node will produce blocks on (use negative value to indicate unlimited) - --max-block-cpu-usage-threshold-us Threshold of CPU block production to - consider block full; when within threshold - of max-block-cpu-usage block can be - produced immediately. Default value 5000 - --max-block-net-usage-threshold-bytes Threshold of NET block production to - consider block full; when within threshold - of max-block-net-usage block can be produced - immediately. Default value 1024 -p [ --producer-name ] arg ID of producer controlled by this node (e.g. inita; may specify multiple times) @@ -64,7 +56,7 @@ Config Options for eosio::producer_plugin: form : - is KEY, or KEOSD + is KEY, KEOSD, or SE KEY: is a string form of a valid EOSIO @@ -77,32 +69,60 @@ Config Options for eosio::producer_plugin: and the approptiate wallet(s) are unlocked - --keosd-provider-timeout arg (=5) Limits the maximum time (in - milliseconds) that is allowed for - sending blocks to a keosd provider for - signing + + SE: indicates the key + resides in Secure + Enclave --greylist-account arg account that can not access to extended CPU/NET virtual resources - --produce-time-offset-us arg (=0) offset of non last block producing time - in microseconds. Negative number - results in blocks to go out sooner, and - positive number results in blocks to go - out later - --last-block-time-offset-us arg (=0) offset of last block producing time in - microseconds. Negative number results - in blocks to go out sooner, and - positive number results in blocks to go - out later + --greylist-limit arg (=1000) Limit (between 1 and 1000) on the + multiple that CPU/NET virtual resources + can extend during low usage (only + enforced subjectively; use 1000 to not + enforce any limit) + --produce-time-offset-us arg (=0) Offset of non last block producing time + in microseconds. Valid range 0 .. + -block_time_interval. + --last-block-time-offset-us arg (=-200000) + Offset of last block producing time in + microseconds. Valid range 0 .. + -block_time_interval. + --cpu-effort-percent arg (=80) Percentage of cpu block production time + used to produce block. Whole number + percentages, e.g. 80 for 80% + --last-block-cpu-effort-percent arg (=80) + Percentage of cpu block production time + used to produce last block. Whole + number percentages, e.g. 80 for 80% + --max-block-cpu-usage-threshold-us arg (=5000) + Threshold of CPU block production to + consider block full; when within + threshold of max-block-cpu-usage block + can be produced immediately + --max-block-net-usage-threshold-bytes arg (=1024) + Threshold of NET block production to + consider block full; when within + threshold of max-block-net-usage block + can be produced immediately --max-scheduled-transaction-time-per-block-ms arg (=100) Maximum wall-clock time, in milliseconds, spent retiring scheduled transactions in any block before returning to normal transaction processing. - --incoming-defer-ratio arg (=1) ratio between incoming transactions and + --subjective-cpu-leeway-us arg (=31000) + Time in microseconds allowed for a + transaction that starts with + insufficient CPU quota to complete and + cover its CPU usage. + --incoming-defer-ratio arg (=1) ratio between incoming transactions and deferred transactions when both are - queued for execution - + queued for execution + --incoming-transaction-queue-size-mb arg (=1024) + Maximum size (in MiB) of the incoming + transaction queue. Exceeding this value + will subjectively drop transaction with + resource exhaustion. --producer-threads arg (=2) Number of worker threads in producer thread pool --snapshots-dir arg (="snapshots") the location of the snapshots directory diff --git a/docs/01_nodeos/03_plugins/resource_monitor_plugin/index.md b/docs/01_nodeos/03_plugins/resource_monitor_plugin/index.md new file mode 100644 index 00000000000..070cb331163 --- /dev/null +++ b/docs/01_nodeos/03_plugins/resource_monitor_plugin/index.md @@ -0,0 +1,66 @@ + +## Overview + +The `resource_monitor_plugin` monitors space usage in the computing system where `nodeos` is running. Specifically, every `resource-monitor-interval-seconds` seconds, +it measures the individual space used by each of the file systems mounted +by `data-dir`, `state-dir`, `blocks-log-dir`, `snapshots-dir`, +`state-history-dir`, and `trace-dir`. +When space usage in any of the monitored file system is within `5%` of the threshold +specified by `resource-monitor-space-threshold`, a warning containing the file system +path and percentage of space has used is printed out. +When space usage exceeds the threshold, +if `resource-monitor-not-shutdown-on-threshold-exceeded` is not set, +`nodeos` gracefully shuts down; if `resource-monitor-not-shutdown-on-threshold-exceeded` is set, `nodeos` prints out warnings periodically +until space usage goes under the threshold. + +`resource_monitor_plugin` is always loaded. +## Usage + +```console +# config.ini +plugin = eosio::resource_monitor_plugin +[options] +``` +```sh +# command-line +nodeos ... --plugin eosio::resource_monitor_plugin [options] +``` + +## Configuration Options + +These can be specified from both the `nodeos` command-line or the `config.ini` file: + +```console +Config Options for eosio::resource_monitor_plugin: + + --resource-monitor-interval-seconds arg (=2) + Time in seconds between two consecutive checks + of space usage. Should be between 1 and 300. + --resource-monitor-space-threshold arg (=90) + Threshold in terms of percentage of used space + vs total space. If the used space is within + `5%` of the threshold, a warning is generated. + If the used space is above the threshold and + `resource-monitor-not-shutdown-on-threshold-exceeded` + is enabled, a shutdown is initiated; otherwise + a warning will be continuously printed out. + The value should be between 6 and 99. + --resource-monitor-not-shutdown-on-threshold-exceeded + A switch used to indicate `nodeos` will "not" + shutdown when threshold is exceeded. When not + set, `nodeos` will shutdown. + --resource-monitor-warning-interval arg (=30) + Number of monitor intervals between which a + warning is displayed. For example, if + `resource-monitor-warning-interval` is to 10 + and `resource-monitor-interval-seconds` is 2, + a warning will be displayed every 20 seconds, + even though the space usage is checked every + 2 seconds. This is used to throttle the + number of warnings in the `nodeos` log file. + Should be between 1 and 450. +``` + +## Plugin Dependencies + +* None diff --git a/docs/01_nodeos/03_plugins/state_history_plugin/index.md b/docs/01_nodeos/03_plugins/state_history_plugin/index.md index 4ee27384f5a..847b68ef23b 100644 --- a/docs/01_nodeos/03_plugins/state_history_plugin/index.md +++ b/docs/01_nodeos/03_plugins/state_history_plugin/index.md @@ -36,6 +36,46 @@ Config Options for eosio::state_history_plugin: the location of the state-history directory (absolute path or relative to application data dir) + --state-history-retained-dir arg (="") + the location of the state history + retained directory (absolute path or + relative to state-history dir). + If the value is empty, it is set to the + value of state-history directory. + --state-history-archive-dir arg (="archive") + the location of the state history + archive directory (absolute path or + relative to state-history dir). + If the value is empty, blocks files + beyond the retained limit will be + deleted. + All files in the archive directory are + completely under user's control, i.e. + they won't be accessed by nodeos + anymore. + --state-history-stride arg (=4294967295) + split the state history log files when + the block number is the multiple of the + stride + When the stride is reached, the current + history log and index will be renamed + '*-history--.log/index' + and a new current history log and index + will be created with the most recent + blocks. All files following + this format will be used to construct + an extended history log. + --max-retained-history-files arg (=10) + the maximum number of history file + groups to retain so that the blocks in + those files can be queried. + When the number is reached, the oldest + history file would be moved to archive + dir or deleted if the archive dir is + empty. + The retained history log files should + not be manipulated by users. --trace-history enable trace history --chain-state-history enable chain state history --state-history-endpoint arg (=127.0.0.1:8080) @@ -44,6 +84,10 @@ Config Options for eosio::state_history_plugin: expose this port to your internal network. --trace-history-debug-mode enable debug mode for trace history + --context-free-data-compression arg (=zlib) + compression mode for context free data + in transaction traces. Supported + options are "zlib" and "none" ``` ## Examples diff --git a/docs/01_nodeos/03_plugins/test_control_api_plugin/api-reference/index.md b/docs/01_nodeos/03_plugins/test_control_api_plugin/api-reference/index.md deleted file mode 100644 index 6451c708686..00000000000 --- a/docs/01_nodeos/03_plugins/test_control_api_plugin/api-reference/index.md +++ /dev/null @@ -1 +0,0 @@ - diff --git a/docs/01_nodeos/03_plugins/test_control_api_plugin/index.md b/docs/01_nodeos/03_plugins/test_control_api_plugin/index.md deleted file mode 100644 index a2ec6d832c2..00000000000 --- a/docs/01_nodeos/03_plugins/test_control_api_plugin/index.md +++ /dev/null @@ -1,46 +0,0 @@ - -## Description - -The `test_control_api_plugin` allows to send a control message to the [test_control_plugin](../test_control_plugin/index.md) telling the plugin to shut down the `nodeos` instance when reaching a particular block. It is intended for testing. - -## Usage - -```console -# config.ini -plugin = eosio::test_control_api_plugin -``` -```sh -# command-line -nodeos ... --plugin eosio::test_control_api_plugin -``` - -## Options - -None - -## Usage Example - -```sh -curl %s/v1/test_control/kill_node_on_producer -d '{ \"producer\":\"%s\", \"where_in_sequence\":%d, \"based_on_lib\":\"%s\" }' -X POST -H \"Content-Type: application/json\"" % -``` - -## Dependencies - -* [`test_control_plugin`](../test_control_plugin/index.md) -* [`chain_plugin`](../chain_plugin/index.md) -* [`http_plugin`](../http_plugin/index.md) - -### Load Dependency Examples - -```console -# config.ini -plugin = eosio::chain_plugin -[options] -plugin = eosio::http_plugin -[options] -``` -```sh -# command-line -nodeos ... --plugin eosio::chain_plugin [operations] [options] \ - --plugin eosio::http_plugin [options] -``` diff --git a/docs/01_nodeos/03_plugins/test_control_plugin/index.md b/docs/01_nodeos/03_plugins/test_control_plugin/index.md deleted file mode 100644 index 3802c8f5b56..00000000000 --- a/docs/01_nodeos/03_plugins/test_control_plugin/index.md +++ /dev/null @@ -1,37 +0,0 @@ - -## Description - -The `test_control_plugin` is designed to cause a graceful shutdown when reaching a particular block in a sequence of blocks produced by a specific block producer. It can be invoked to either shutdown on the **head block** or the **last irreversible block**. - -This is intended for testing, to determine exactly when a `nodeos` instance will shutdown. - -## Usage - -```console -# config.ini -plugin = eosio::test_control_plugin -``` -```sh -# command-line -nodeos ... --plugin eosio::test_control_plugin -``` - -## Options - -None - -## Dependencies - -* [`chain_plugin`](../chain_plugin/index.md) - -### Load Dependency Examples - -```console -# config.ini -plugin = eosio::chain_plugin -[options] -``` -```sh -# command-line -nodeos ... --plugin eosio::chain_plugin [operations] [options] -``` diff --git a/docs/01_nodeos/04_replays/how-to-replay-from-a-blocks.log.md b/docs/01_nodeos/04_replays/how-to-replay-from-a-blocks.log.md index 9c756cd9f74..da9191bdb28 100644 --- a/docs/01_nodeos/04_replays/how-to-replay-from-a-blocks.log.md +++ b/docs/01_nodeos/04_replays/how-to-replay-from-a-blocks.log.md @@ -9,8 +9,8 @@ The table below sumarizes the actions you should take for each of the files enum Folder name | File name | Action ----------------------- | ------------------ | ------ data/blocks | blocks.index | Remove -data/blocks | blocks.log | Replace this file with the `block.log` you want to replay -data/blocks/reversible | forkdb.dat | Remove +data/blocks | blocks.log | Replace this file with the `blocks.log` you want to replay +data/state | fork_db.dat | Remove data/blocks/reversible | shared_memory.bin | Remove data/blocks/reversible | shared_memory.meta | Remove diff --git a/docs/01_nodeos/05_rpc_apis/index.md b/docs/01_nodeos/05_rpc_apis/index.md index 591bccd5571..0329aaa6335 100644 --- a/docs/01_nodeos/05_rpc_apis/index.md +++ b/docs/01_nodeos/05_rpc_apis/index.md @@ -7,5 +7,4 @@ link_text: RPC APIs * [DB Size API Reference](../03_plugins/db_size_api_plugin/api-reference/index.md) * [Net API Reference](../03_plugins/net_api_plugin/api-reference/index.md) * [Producer API Reference](../03_plugins/producer_api_plugin/api-reference/index.md) -* [Test Control API Reference](../03_plugins/test_control_api_plugin/api-reference/index.md) * [Trace API Reference](../03_plugins/trace_api_plugin/api-reference/index.md) diff --git a/docs/01_nodeos/06_logging/01_logging-levels.md b/docs/01_nodeos/06_logging/01_logging-levels.md deleted file mode 100644 index bb68ceeeb1d..00000000000 --- a/docs/01_nodeos/06_logging/01_logging-levels.md +++ /dev/null @@ -1,66 +0,0 @@ ---- -content_title: Logging Levels ---- - -There are six available logging levels: -- all -- debug -- info -- warn -- error -- off - -Sample `logging.json`: - -``` -{ - "includes": [], - "appenders": [{ - "name": "consoleout", - "type": "console", - "args": { - "stream": "std_out", - "level_colors": [{ - "level": "debug", - "color": "green" - },{ - "level": "warn", - "color": "brown" - },{ - "level": "error", - "color": "red" - } - ] - }, - "enabled": true - },{ - "name": "net", - "type": "gelf", - "args": { - "endpoint": "10.10.10.10", - "host": "test" - }, - "enabled": true - } - ], - "loggers": [{ - "name": "default", - "level": "info", - "enabled": true, - "additivity": false, - "appenders": [ - "consoleout", - "net" - ] - },{ - "name": "net_plugin_impl", - "level": "debug", - "enabled": true, - "additivity": false, - "appenders": [ - "net" - ] - } - ] -} -``` diff --git a/docs/01_nodeos/06_logging/00_setup-logging.json.md b/docs/01_nodeos/06_logging/10_native_logging/10_setup-logging.json.md similarity index 100% rename from docs/01_nodeos/06_logging/00_setup-logging.json.md rename to docs/01_nodeos/06_logging/10_native_logging/10_setup-logging.json.md diff --git a/docs/01_nodeos/06_logging/10_native_logging/20_logging-levels.md b/docs/01_nodeos/06_logging/10_native_logging/20_logging-levels.md new file mode 100644 index 00000000000..3c1e4260e65 --- /dev/null +++ b/docs/01_nodeos/06_logging/10_native_logging/20_logging-levels.md @@ -0,0 +1,168 @@ +--- +content_title: Logging Levels +--- + +There are six available logging levels: +- all +- debug +- info +- warn +- error +- off + +Sample `logging.json`: + +``` +{ + "includes": [], + "appenders": [{ + "name": "stderr", + "type": "console", + "args": { + "stream": "std_error", + "level_colors": [{ + "level": "debug", + "color": "green" + },{ + "level": "warn", + "color": "brown" + },{ + "level": "error", + "color": "red" + } + ], + "flush": true + }, + "enabled": true + },{ + "name": "stdout", + "type": "console", + "args": { + "stream": "std_out", + "level_colors": [{ + "level": "debug", + "color": "green" + },{ + "level": "warn", + "color": "brown" + },{ + "level": "error", + "color": "red" + } + ], + "flush": true + }, + "enabled": true + },{ + "name": "net", + "type": "gelf", + "args": { + "endpoint": "10.10.10.10:12201", + "host": "host_name" + }, + "enabled": true + },{ + "name": "zip", + "type": "zipkin", + "args": { + "endpoint": "http://127.0.0.1:9411", + "path": "/api/v2/spans", + "service_name": "nodeos", + "timeout_us": 200000 + }, + "enabled": true + } + ], + "loggers": [{ + "name": "default", + "level": "debug", + "enabled": true, + "additivity": false, + "appenders": [ + "stderr", + "net" + ] + },{ + "name": "zipkin", + "level": "debug", + "enabled": true, + "additivity": false, + "appenders": [ + "zip" + ] + },{ + "name": "net_plugin_impl", + "level": "info", + "enabled": true, + "additivity": false, + "appenders": [ + "stderr", + "net" + ] + },{ + "name": "http_plugin", + "level": "debug", + "enabled": true, + "additivity": false, + "appenders": [ + "stderr", + "net" + ] + },{ + "name": "producer_plugin", + "level": "debug", + "enabled": true, + "additivity": false, + "appenders": [ + "stderr", + "net" + ] + },{ + "name": "transaction_success_tracing", + "level": "debug", + "enabled": true, + "additivity": false, + "appenders": [ + "stderr", + "net" + ] + },{ + "name": "transaction_failure_tracing", + "level": "debug", + "enabled": true, + "additivity": false, + "appenders": [ + "stderr", + "net" + ] + },{ + "name": "state_history", + "level": "debug", + "enabled": true, + "additivity": false, + "appenders": [ + "stderr", + "net" + ] + },{ + "name": "trace_api", + "level": "debug", + "enabled": true, + "additivity": false, + "appenders": [ + "stderr", + "net" + ] + },{ + "name": "blockvault_client_plugin", + "level": "debug", + "enabled": true, + "additivity": false, + "appenders": [ + "stderr", + "net" + ] + } + ] +} +``` diff --git a/docs/01_nodeos/06_logging/10_native_logging/index.md b/docs/01_nodeos/06_logging/10_native_logging/index.md new file mode 100644 index 00000000000..7828219f402 --- /dev/null +++ b/docs/01_nodeos/06_logging/10_native_logging/index.md @@ -0,0 +1,113 @@ +--- +content_title: Native Logging +link_text: Native Logging +--- + +Logging for `nodeos` is controlled by the `logging.json` file. CLI options can be passed to `nodeos` to [setup `logging.json`](10_setup-logging.json.md). The logging configuration file can be used to define [appenders](#appenders) and tie them to [loggers](#loggers) and [logging levels](20_logging-levels.md). + +## Appenders + +The logging library built into EOSIO supports two appender types: + +- [Console](#console) +- [GELF](#gelf) (Graylog Extended Log Format) + +### Console + +This will output log messages to the screen. The configuration options are: + +- `name` - arbitrary name to identify instance for use in loggers +- `type` - "console" +- `stream` - "std_out" or "std_err" +- `level_colors` - maps a log level to a colour + - level - see [logging levels](20_logging-levels.md) + - color - may be one of ("red", "green", "brown", "blue", "magenta", "cyan", "white", "console_default") +- `enabled` - bool value to enable/disable the appender. + +Example: + +```json +{ + "name": "consoleout", + "type": "console", + "args": { + "stream": "std_out", + + "level_colors": [{ + "level": "debug", + "color": "green" + },{ + "level": "warn", + "color": "brown" + },{ + "level": "error", + "color": "red" + } + ] + }, + "enabled": true +} +``` + +### GELF + +This sends the log messages to `Graylog`. `Graylog` is a fully integrated platform for collecting, indexing, and analyzing log messages. The configuration options are: + + - `name` - arbitrary name to identify instance for use in loggers + - `type` - "gelf" + - `endpoint` - ip address and port number + - `host` - Graylog hostname, identifies you to Graylog. + - `enabled` - bool value to enable/disable the appender. + +Example: + +```json +{ + "name": "net", + "type": "gelf", + "args": { + "endpoint": "104.198.210.18:12202”, + "host": + }, + "enabled": true +} +``` + +## Loggers + +The logging library built into EOSIO currently supports the following loggers: + +- `default` - the default logger, always enabled. +- `net_plugin_impl` - detailed logging for the net plugin. +- `http_plugin` - detailed logging for the http plugin. +- `producer_plugin` - detailed logging for the producer plugin. +- `state_history` - detailed logging for state history plugin. +- `transaction_success_tracing` - detailed log that emits successful verdicts from relay nodes on the P2P network. +- `transaction_failure_tracing` - detailed log that emits failed verdicts from relay nodes on the P2P network. +- `trace_api` - detailed logging for the trace_api plugin. +- `blockvault_client_plugin` - detailed logging for the blockvault client plugin. + +The configuration options are: + + - `name` - must match one of the names described above. + - `level` - see logging levels below. + - `enabled` - bool value to enable/disable the logger. + - `additivity` - true or false + - `appenders` - list of appenders by name (name in the appender configuration) + +Example: + +```json +{ + "name": "net_plugin_impl", + "level": "debug", + "enabled": true, + "additivity": false, + "appenders": [ + "net" + ] +} +``` + +[[info]] +| The default logging level for all loggers if no `logging.json` is provided is `info`. Each logger can be configured independently in the `logging.json` file. diff --git a/docs/01_nodeos/06_logging/20_third_party_logging/10_deep_mind_logger.md b/docs/01_nodeos/06_logging/20_third_party_logging/10_deep_mind_logger.md new file mode 100644 index 00000000000..ce13dba2bde --- /dev/null +++ b/docs/01_nodeos/06_logging/20_third_party_logging/10_deep_mind_logger.md @@ -0,0 +1,44 @@ +--- +content_title: Deep-mind Logger Integration +link_text: Deep-mind Logger Integration +--- + +## Overview + +The `Deep-mind logger` is part of the `dfuse` [platform]([https://dfuse.io/](https://dfuse.io/)) which is a highly scalable and performant [open-source]([https://github.com/dfuse-io/dfuse-eosio/tree/master](https://github.com/dfuse-io/dfuse-eosio/tree/master)) platform for searching and processing blockchain data. + +### How To Enable Deep-mind Logger + +EOSIO integrates the `nodeos` core service daemon with `deep-mind logger`. To benefit from full `deep-mind` logging functionality you must start your `nodeos` instance with the flag `--deep-mind`. After the start you can observe in the `nodeos` console output the informative details outputs created by the `deep-mind` logger. They distinguish themselves from the default `nodeos` output lines because they start with the `DMLOG` keyword. + +Examples of `deep-mind` log lines as you would see them in the `nodeos` output console: + +```console +DMLOG START_BLOCK 30515 + +DMLOG TRX_OP CREATE onblock 308f77bf49ab4ddde74d37c7310c0742e253319d9da57ebe51eb7b35f1ffe174 {"expiration":"2020-11-12T10:13:06","ref_block_num":30514,...} + +DMLOG CREATION_OP ROOT 0 + +DMLOG RLIMIT_OP ACCOUNT_USAGE UPD {"owner":"eosio","net_usage":{"last_ordinal":1316982371,"value_ex":0,"consumed":0},"cpu_usage":{"last_ordinal":1316982371,"value_ex":24855,"consumed":101},"ram_usage":27083} + +DMLOG APPLIED_TRANSACTION 30515 {"id":"308f77bf49ab4ddde74d37c7310c0742e253319d9da57ebe51eb7b35f1ffe174","block_num":30515,"block_time":"2020-11-12T10:13:05.500",...} + +DMLOG RLIMIT_OP STATE UPD {"average_block_net_usage":{"last_ordinal":30514,"value_ex":0,"consumed":0},"average_block_cpu_usage":{"last_ordinal":30514,...} +DMLOG ACCEPTED_BLOCK 30516 {"block_num":30516,"dpos_proposed_irreversible_blocknum":30516,"dpos_irreversible_blocknum":30515,... + +... + +DMLOG FEATURE_OP ACTIVATE 0ec7e080177b2c02b278d5088611686b49d739925a92d9bfcacd7fc6b74053bd {"feature_digest":"0ec7e080177b2c02b278d5088611686b49d739925a92d9bfcacd7fc6b74053bd","subjective_restrictions":{"enabled":true,"preactivation_required":false,"earliest_allowed_activation_time":"1970-01-01T00:00:00.000"},"description_digest":"64fe7df32e9b86be2b296b3f81dfd527f84e82b98e363bc97e40bc7a83733310","dependencies":[],"protocol_feature_type":"builtin","specification": +[{"name":"builtin_feature_codename","value":"PREACTIVATE_FEATURE"}]} + +... + +DMLOG FEATURE_OP ACTIVATE 825ee6288fb1373eab1b5187ec2f04f6eacb39cb3a97f356a07c91622dd61d16 {"feature_digest":"825ee6288fb1373eab1b5187ec2f04f6eacb39cb3a97f356a07c91622dd61d16","subjective_restrictions":{"enabled":true,"preactivation_required":true,"earliest_allowed_activation_time":"1970-01-01T00:00:00.000"},"description_digest":"14cfb3252a5fa3ae4c764929e0bbc467528990c9cc46aefcc7f16367f28b6278","dependencies":[],"protocol_feature_type":"builtin","specification": +[{"name":"builtin_feature_codename","value":"KV_DATABASE"}]} + +... + +DMLOG FEATURE_OP ACTIVATE c3a6138c5061cf291310887c0b5c71fcaffeab90d5deb50d3b9e687cead45071 {"feature_digest":"c3a6138c5061cf291310887c0b5c71fcaffeab90d5deb50d3b9e687cead45071","subjective_restrictions":{"enabled":true,"preactivation_required":true,"earliest_allowed_activation_time":"1970-01-01T00:00:00.000"},"description_digest":"69b064c5178e2738e144ed6caa9349a3995370d78db29e494b3126ebd9111966","dependencies":[],"protocol_feature_type":"builtin","specification": +[{"name":"builtin_feature_codename","value":"ACTION_RETURN_VALUE"}]} +``` diff --git a/docs/01_nodeos/06_logging/20_third_party_logging/20_zipkin_tracer.md b/docs/01_nodeos/06_logging/20_third_party_logging/20_zipkin_tracer.md new file mode 100644 index 00000000000..fef052ab791 --- /dev/null +++ b/docs/01_nodeos/06_logging/20_third_party_logging/20_zipkin_tracer.md @@ -0,0 +1,34 @@ +--- +content_title: Zipkin Tracer Integration +link_text: Zipkin Tracer Integration +--- + +## Overview + +The `Zipkin service` is a [distributed tracing system](https://zipkin.io/). It helps gather timing data needed to troubleshoot latency problems in service architectures. Its features include both the collection and lookup of this data. `Zipkin tracer` is the EOSIO component that sends traces to the `Zipkin service`. The `Zipkin` service can be installed in the local environment or it can be remote. + +### How To Enable Zipkin Tracer + +EOSIO makes available `Zipkin tracer` through the [core `chain_plugin`](../../03_plugins/chain_plugin). To enable the `Zipkin tracer` you must set the `telemetry-url` parameter for the `chain_plugin`. There are two additional parameters you can set: `telemetry-service-name` and `telemetry-timeout-us`. All three available parameters are detailed below: + +* `telemetry-url` specifies the url of the Zipkin service, e.g. [http://127.0.0.1:9411/api/v2/spans](http://127.0.0.1:9411/api/v2/spans) if it is installed in the local environment. +* `telemetry-service-name` specifies the Zipkin `localEndpoint.serviceName` sent with each span. +* `telemetry-timeout-us`, optional parameter, specifies the timeout for sending zipkin spans with default value set to 200000. `Zipkin tracer` sends zipkin spans, in json format, to the `Zipkin service` via HTTP POST requests on a separate thread. `telemetry-timeout-us` specifies how long to allow each HTTP POST request before reaching the time out and thus fail. Nine consecutive failures will disable the `Zipkin tracer` until a restart of `nodeos` daemon occurs. + +Once `Zipkin tracer` is enabled you can observe the detailed traces it sends to the `Zipkin service`. Examples of such traces are given below. + +Submitted transaction: + +![Submitted transaction](submitted_transaction.png) + +Processed transaction: + +![Processed transaction](processed_transaction.png) + +Block accepted: + +![Block accepted](block_accepted.png) + +SHiP Accepted: + +![SHiP Accepted](ship_accepted.png) diff --git a/docs/01_nodeos/06_logging/20_third_party_logging/block_accepted.png b/docs/01_nodeos/06_logging/20_third_party_logging/block_accepted.png new file mode 100644 index 00000000000..baced1d2b3a Binary files /dev/null and b/docs/01_nodeos/06_logging/20_third_party_logging/block_accepted.png differ diff --git a/docs/01_nodeos/06_logging/20_third_party_logging/index.md b/docs/01_nodeos/06_logging/20_third_party_logging/index.md new file mode 100644 index 00000000000..ce689a876d5 --- /dev/null +++ b/docs/01_nodeos/06_logging/20_third_party_logging/index.md @@ -0,0 +1,15 @@ +--- +content_title: Third-Party Logging And Tracing Integration +link_text: Third-Party Logging And Tracing Integration +--- + +## Overview + +To stay informed about the overall and detailed performance of your EOSIO-based blockchain node(s), you can make use of the telemetry tools available. EOSIO offers integration with two such telemetry tools: + +* [Deep-mind logger](10_deep_mind_logger.md) +* [Zipkin tracer](20_zipkin_tracer.md) + +## Performance Considerations + +Many wise people already said it, everything comes with a price. The telemetry tools, when activated, will have a certain impact on your `nodeos` performance which depends on various factors, but nevertheless the performance will be impacted; therefore, it is recommended you use them wisely in those situations when you really need the extra detailed information they provide, and then you turn them off. diff --git a/docs/01_nodeos/06_logging/20_third_party_logging/processed_transaction.png b/docs/01_nodeos/06_logging/20_third_party_logging/processed_transaction.png new file mode 100644 index 00000000000..baced1d2b3a Binary files /dev/null and b/docs/01_nodeos/06_logging/20_third_party_logging/processed_transaction.png differ diff --git a/docs/01_nodeos/06_logging/20_third_party_logging/ship_accepted.png b/docs/01_nodeos/06_logging/20_third_party_logging/ship_accepted.png new file mode 100644 index 00000000000..baced1d2b3a Binary files /dev/null and b/docs/01_nodeos/06_logging/20_third_party_logging/ship_accepted.png differ diff --git a/docs/01_nodeos/06_logging/20_third_party_logging/submitted_transaction.png b/docs/01_nodeos/06_logging/20_third_party_logging/submitted_transaction.png new file mode 100644 index 00000000000..baced1d2b3a Binary files /dev/null and b/docs/01_nodeos/06_logging/20_third_party_logging/submitted_transaction.png differ diff --git a/docs/01_nodeos/06_logging/index.md b/docs/01_nodeos/06_logging/index.md index 8a7670e7fb5..1d6dcbab046 100644 --- a/docs/01_nodeos/06_logging/index.md +++ b/docs/01_nodeos/06_logging/index.md @@ -1,110 +1,9 @@ --- -content_title: Nodeos Logging +content_title: Logging And Tracing +link_text: Logging And Tracing --- -Logging for `nodeos` is controlled by the `logging.json` file. CLI options can be passed to `nodeos` to [setup `logging.json`](00_setup-logging.json.md). The logging configuration file can be used to define [appenders](#appenders) and tie them to [loggers](#loggers) and [logging levels](01_logging-levels.md). +Nodeos has the following logging and tracing features: -## Appenders - -The logging library built into EOSIO supports two appender types: - -- [Console](#console) -- [GELF](#gelf) (Graylog Extended Log Format) - -### Console - -This will output log messages to the screen. The configuration options are: - -- `name` - arbitrary name to identify instance for use in loggers -- `type` - "console" -- `stream` - "std_out" or "std_err" -- `level_colors` - maps a log level to a colour - - level - see [logging levels](01_logging-levels.md) - - color - may be one of ("red", "green", "brown", "blue", "magenta", "cyan", "white", "console_default") -- `enabled` - bool value to enable/disable the appender. - -Example: - -```json -{ - "name": "consoleout", - "type": "console", - "args": { - "stream": "std_out", - - "level_colors": [{ - "level": "debug", - "color": "green" - },{ - "level": "warn", - "color": "brown" - },{ - "level": "error", - "color": "red" - } - ] - }, - "enabled": true -} -``` - -### GELF - -This sends the log messages to `Graylog`. `Graylog` is a fully integrated platform for collecting, indexing, and analyzing log messages. The configuration options are: - - - `name` - arbitrary name to identify instance for use in loggers - - `type` - "gelf" - - `endpoint` - ip address and port number - - `host` - Graylog hostname, identifies you to Graylog. - - `enabled` - bool value to enable/disable the appender. - -Example: - -```json -{ - "name": "net", - "type": "gelf", - "args": { - "endpoint": "104.198.210.18:12202”, - "host": - }, - "enabled": true -} -``` - -## Loggers - -The logging library built into EOSIO currently supports the following loggers: - -- `default` - the default logger, always enabled. -- `net_plugin_impl` - detailed logging for the net plugin. -- `http_plugin` - detailed logging for the http plugin. -- `producer_plugin` - detailed logging for the producer plugin. -- `transaction_tracing` - detailed log that emits verdicts from relay nodes on the P2P network. -- `transaction_failure_tracing` - detailed log that emits failed verdicts from relay nodes on the P2P network. -- `trace_api` - detailed logging for the trace_api plugin. - -The configuration options are: - - - `name` - must match one of the names described above. - - `level` - see logging levels below. - - `enabled` - bool value to enable/disable the logger. - - `additivity` - true or false - - `appenders` - list of appenders by name (name in the appender configuration) - -Example: - -```json -{ - "name": "net_plugin_impl", - "level": "debug", - "enabled": true, - "additivity": false, - "appenders": [ - "net" - ] -} -``` - -[[info]] -| The default logging level for all loggers if no `logging.json` is provided is `info`. Each logger can be configured independently in the `logging.json` file. +* [Native Logging](10_native_logging/index.md) +* [Third-Party Logging And Tracing Integration](20_third_party_logging/index.md) diff --git a/docs/01_nodeos/07_concepts/05_storage-and-read-modes.md b/docs/01_nodeos/07_concepts/05_storage-and-read-modes.md index df2a872729a..4de395bd377 100644 --- a/docs/01_nodeos/07_concepts/05_storage-and-read-modes.md +++ b/docs/01_nodeos/07_concepts/05_storage-and-read-modes.md @@ -2,22 +2,33 @@ content_title: Storage and Read Modes --- -The EOSIO platform stores blockchain information in various data structures at various stages of a transaction's lifecycle. Some of these are described below. The producing node is the `nodeos` instance run by the block producer who is currently creating blocks for the blockchain (which changes every 6 seconds, producing 12 blocks in sequence before switching to another producer.) +The EOSIO platform stores blockchain information in various data structures at various stages of a transaction's lifecycle. Some of these are described below. The producing node is the `nodeos` instance run by the block producer who is currently creating blocks for the blockchain (which changes every 6 seconds, producing 12 blocks in sequence before switching to another producer). ## Blockchain State and Storage -Every `nodeos` instance creates some internal files to housekeep the blockchain state. These files reside in the `~/eosio/nodeos/data` installation directory and their purpose is described below: +Every `nodeos` instance creates some internal files to store the blockchain state. These files reside in the `~/eosio/nodeos/data` installation directory and their purpose is described below: * The `blocks.log` is an append only log of blocks written to disk and contains all the irreversible blocks. These blocks contain final, confirmed transactions. * `reversible_blocks` is a memory mapped file and contains blocks that have been written to the blockchain but have not yet become irreversible. These blocks contain valid pushed transactions that still await confirmation to become final via the consensus protocol. The head block is the last block written to the blockchain, stored in `reversible_blocks`. -* The `chain state` or `chain database` is currently stored and cached in a memory mapped file. It contains the blockchain state associated with each block, including account details, deferred transactions, and data stored using multi index tables in smart contracts. The last 65,536 block IDs are also cached to support Transaction as Proof of Stake (TaPOS). The transaction ID/expiration is also cached until the transaction expires. - +* The `chain state` or `chain database` is stored either in `chainbase` or in `rocksdb`, dependant on the `nodeos` `chain_plugin` configuration option `backing-store`. It contains the blockchain state associated with each block, including account details, deferred transactions, and data stored using multi index tables in smart contracts. The last 65,536 block IDs are also cached to support Transaction as Proof of Stake (TaPOS). The transaction ID/expiration is also cached until the transaction expires. * The `pending block` is an in memory block containing transactions as they are processed and pushed into the block; this will/may eventually become the head block. If the `nodeos` instance is the producing node, the pending block is distributed to other `nodeos` instances. -* Outside the chain state, block data is cached in RAM until it becomes final/irreversible; especifically the signed block itself. After the last irreversible block (LIB) catches up to the block, that block is then retrieved from the irreversible blocks log. +* Outside the `chain state`, block data is cached in RAM until it becomes final/irreversible; specifically the signed block itself. After the last irreversible block (LIB) catches up to the block, that block is then retrieved from the irreversible blocks log. + +### Configurable state storage + +`Nodeos` stores the transaction history and current state. The transaction history is stored in the `blocks.log` file on disk. Current state, which is changed by the execution of transactions, is currently stored using chainbase or RocksDB (as of EOSIO 2.1). EOSIO 2.1 introduces configurable state storage and currently supports these backing stores: + +* Chainbase +* RocksDB + +Chainbase is a proprietary in-memory transactional database, built by Block.one, which uses memory mapped files for persistence. + +RocksDB is an open source persistent key value store. Storing state in memory is fast, however limited by the amount of available RAM. RocksDB utilises low latency storage such as flash drives and high-speed disk drives to persist data and memory caches for fast data access. For some deployments, RocksDB may be a better state store. See [the RocksDB website](https://rocksdb.org/) for more information. + ## EOSIO Interfaces -EOSIO provides a set of [services](../../) and [interfaces](https://developers.eos.io/manuals/eosio.cdt/latest/files) that enable contract developers to persist state across action, and consequently transaction, boundaries. Contracts may use these services and interfaces for various purposes. For example, `eosio.token` contract keeps balances for all users in the chain database. Each instance of `nodeos` keeps the database in memory, so contracts can read and write data with ease. +EOSIO provides a set of [services](../../) and [interfaces](https://developers.eos.io/manuals/eosio.cdt/latest/files) that enable contract developers to persist state across action, and consequently transaction, boundaries. Contracts may use these services and interfaces for various purposes. For example, `eosio.token` contract keeps balances for all users in the `chain database`. Each instance of `nodeos` maintains the `chain database` in an efficient data store, so contracts can read and write data with ease. ### Nodeos RPC API diff --git a/docs/01_nodeos/07_concepts/10_context-free-data/05_how-to-prune-context-free-data.md b/docs/01_nodeos/07_concepts/10_context-free-data/05_how-to-prune-context-free-data.md new file mode 100644 index 00000000000..aaed758d3a9 --- /dev/null +++ b/docs/01_nodeos/07_concepts/10_context-free-data/05_how-to-prune-context-free-data.md @@ -0,0 +1,376 @@ +--- +content_title: How to prune context-free data +link_text: How to prune context-free data +--- + +## Summary + +This how-to procedure showcases the steps to prune context-free data (CFD) from a transaction. The process involves launching the [`eosio-blocklog`](../../../10_utilities/eosio-blocklog.md) utility with the `--prune-transactions` option, the transaction ID(s) that contain(s) the context-free data, and additional options as specified below. + +[[caution | Data Pruning on Public Chains]] +| Pruning transaction data is not suitable for public EOSIO blockchains, unless previously agreed upon through EOSIO consensus by a supermajority of producers. Even if a producing node on a public EOSIO network prunes context-free data from a transaction, only their node would be affected. The integrity of the blockchain would not be compromised. + +## Prerequisites + +The following items must be known in advance or completed before starting the procedure: + +* The ID of a retired transaction with context-free data in a finalized block. +* Become familiar with the [context-free data](index.md) section of a transaction. +* Review the [`eosio-blocklog`](../../../10_utilities/eosio-blocklog.md) command-line utility reference. + +## Procedure + +Complete the following steps to prune the context-free data from the transaction: + +1. Locate the transaction ID you want to prune the context-free data from, e.g. ``. The transaction ID can also be found on the `id` field of the transaction. +2. Locate the block number that contains the transaction, e.g. ``. Make sure the block number matches the `block_num` field of the transaction. +3. Find the blocks directory and state history directory (if applicable), e.g. `` and ``, respectively. +4. Launch the [`eosio-blocklog`](../../../10_utilities/eosio-blocklog.md) utility as follows: + + `eosio-blocklog [--blocks-dir ] [--state-history-dir ] --prune-transactions --block-num --transaction [--transaction ...]` + + If the operation is *successful*: + * The [`eosio-blocklog`](../../../10_utilities/eosio-blocklog.md) utility terminates silently with a zero error code (no error). + * The following fields are updated within the pruned transaction from the block logs: + * The `prunable_data["prunable_data"][0]` field is set from 0 to 1. + * The `signatures` field is set to an empty array. + * The `context_free_data` field is set to an empty array. + * The `packed_context_free_data` field, if any, is removed. + + If the operaton is *unsuccessful*: + * The [`eosio-blocklog`](../../../10_utilities/eosio-blocklog.md) utility outputs an error to `stderr` and terminates with a non-zero error code (indicating an error). + +## Notes + +Some additional considerations are in order: + +* You can pass multiple transactions to [`eosio-blocklog`](../../../10_utilities/eosio-blocklog.md) if they are within the same block. +* You can use [`eosio-blocklog`](../../../10_utilities/eosio-blocklog.md) to display the block that contains the pruned transactions. + +## Example + +In this example, we reproduce the steps listed in the [Procedure](#procedure) section above. + +### Transaction sample (before) + +Refer to the following transaction sample with context-free data: + +```json +{ + "id": "1b9a9c53f9b692d3382bcc19c0c21eb22207e2f51a30fe88dabbb45376b6ff23", + "trx": { + "receipt": { + "status": "executed", + "cpu_usage_us": 155, + "net_usage_words": 14, + "trx": [ + 1, + { + "compression": "none", + "prunable_data": { + "prunable_data": [ + 0, + { + "signatures": [ + "SIG_K1_K3AJXEMFH99KScLFC1cnLA3WDnVK7WRsS8BtafHfP4VWmfQXXwX21KATVVtrCqopkcve6V8noc5bS4BJkwgSsonpfpWEJi" + ], + "packed_context_free_data": "0203a1b2c3031a2b3c" + } + ] + }, + "packed_trx": "ec42545f7500ffe8aa290000000100305631191abda90000000000901d4d00000100305631191abda90000000000901d4d0100305631191abda900000000a8ed32320000" + } + ] + }, + "trx": { + "expiration": "2020-09-06T02:01:16", + "ref_block_num": 117, + "ref_block_prefix": 699066623, + "max_net_usage_words": 0, + "max_cpu_usage_ms": 0, + "delay_sec": 0, + "context_free_actions": [ + { + "account": "payloadless", + "name": "doit", + "authorization": [], + "data": "" + } + ], + "actions": [ + { + "account": "payloadless", + "name": "doit", + "authorization": [ + { + "actor": "payloadless", + "permission": "active" + } + ], + "data": "" + } + ], + "signatures": [ + "SIG_K1_K3AJXEMFH99KScLFC1cnLA3WDnVK7WRsS8BtafHfP4VWmfQXXwX21KATVVtrCqopkcve6V8noc5bS4BJkwgSsonpfpWEJi" + ], + "context_free_data": [ + "a1b2c3", + "1a2b3c" + ] + } + }, + "block_time": "2020-09-06T02:00:47.000", + "block_num": 119, + "last_irreversible_block": 128, + "traces": [ + { + "action_ordinal": 1, + "creator_action_ordinal": 0, + "closest_unnotified_ancestor_action_ordinal": 0, + "receipt": { + "receiver": "payloadless", + "act_digest": "4f09a630d4456585ee4ec5ef96c14151587367ad381f9da445b6b6239aae82cf", + "global_sequence": 156, + "recv_sequence": 2, + "auth_sequence": [], + "code_sequence": 1, + "abi_sequence": 1 + }, + "receiver": "payloadless", + "act": { + "account": "payloadless", + "name": "doit", + "authorization": [], + "data": "" + }, + "context_free": true, + "elapsed": 206, + "console": "Im a payloadless action", + "trx_id": "1b9a9c53f9b692d3382bcc19c0c21eb22207e2f51a30fe88dabbb45376b6ff23", + "block_num": 119, + "block_time": "2020-09-06T02:00:47.000", + "producer_block_id": null, + "account_ram_deltas": [], + "account_disk_deltas": [], + "except": null, + "error_code": null, + "return_value_hex_data": "" + }, + { + "action_ordinal": 2, + "creator_action_ordinal": 0, + "closest_unnotified_ancestor_action_ordinal": 0, + "receipt": { + "receiver": "payloadless", + "act_digest": "b8871e8f3c79b02804a2ad28acb015f503e7f6e56f35565e5fa37b6767da1aa5", + "global_sequence": 157, + "recv_sequence": 3, + "auth_sequence": [ + [ + "payloadless", + 3 + ] + ], + "code_sequence": 1, + "abi_sequence": 1 + }, + "receiver": "payloadless", + "act": { + "account": "payloadless", + "name": "doit", + "authorization": [ + { + "actor": "payloadless", + "permission": "active" + } + ], + "data": "" + }, + "context_free": false, + "elapsed": 11, + "console": "Im a payloadless action", + "trx_id": "1b9a9c53f9b692d3382bcc19c0c21eb22207e2f51a30fe88dabbb45376b6ff23", + "block_num": 119, + "block_time": "2020-09-06T02:00:47.000", + "producer_block_id": null, + "account_ram_deltas": [], + "account_disk_deltas": [], + "except": null, + "error_code": null, + "return_value_hex_data": "" + } + ] +} +``` + +### Steps + +Using the above transaction to recreate the steps above: + +1. Locate the transaction ID: `1b9a9c53f9b692d3382bcc19c0c21eb22207e2f51a30fe88dabbb45376b6ff23`. +2. Locate the block number: `119`. +3. Find the blocks directory and state history directory (if applicable), e.g. `` and ``. +4. Launch the [`eosio-blocklog`](../../../10_utilities/eosio-blocklog.md) utility as follows: + + `eosio-blocklog --blocks-dir --state-history-dir --prune-transactions --block-num 119 --transaction 1b9a9c53f9b692d3382bcc19c0c21eb22207e2f51a30fe88dabbb45376b6ff23` + +If *successful*, the utility returns silently. If *unsuccessful*, it outputs an error to `stderr`. + +### Transaction sample (after) + +After retrieving the transaction a second time, the pruned transaction looks as follows: + +```json +{ + "id": "1b9a9c53f9b692d3382bcc19c0c21eb22207e2f51a30fe88dabbb45376b6ff23", + "trx": { + "receipt": { + "status": "executed", + "cpu_usage_us": 155, + "net_usage_words": 14, + "trx": [ + 1, + { + "compression": "none", + "prunable_data": { + "prunable_data": [ + 1, + { + "digest": "6f29ea8ab323ffee90585238ff32300c4ee6aa563235ff05f3c1feb855f09189" + } + ] + }, + "packed_trx": "ec42545f7500ffe8aa290000000100305631191abda90000000000901d4d00000100305631191abda90000000000901d4d0100305631191abda900000000a8ed32320000" + } + ] + }, + "trx": { + "expiration": "2020-09-06T02:01:16", + "ref_block_num": 117, + "ref_block_prefix": 699066623, + "max_net_usage_words": 0, + "max_cpu_usage_ms": 0, + "delay_sec": 0, + "context_free_actions": [ + { + "account": "payloadless", + "name": "doit", + "authorization": [], + "data": "" + } + ], + "actions": [ + { + "account": "payloadless", + "name": "doit", + "authorization": [ + { + "actor": "payloadless", + "permission": "active" + } + ], + "data": "" + } + ], + "signatures": [], + "context_free_data": [] + } + }, + "block_time": "2020-09-06T02:00:47.000", + "block_num": 119, + "last_irreversible_block": 131, + "traces": [ + { + "action_ordinal": 1, + "creator_action_ordinal": 0, + "closest_unnotified_ancestor_action_ordinal": 0, + "receipt": { + "receiver": "payloadless", + "act_digest": "4f09a630d4456585ee4ec5ef96c14151587367ad381f9da445b6b6239aae82cf", + "global_sequence": 156, + "recv_sequence": 2, + "auth_sequence": [], + "code_sequence": 1, + "abi_sequence": 1 + }, + "receiver": "payloadless", + "act": { + "account": "payloadless", + "name": "doit", + "authorization": [], + "data": "" + }, + "context_free": true, + "elapsed": 206, + "console": "Im a payloadless action", + "trx_id": "1b9a9c53f9b692d3382bcc19c0c21eb22207e2f51a30fe88dabbb45376b6ff23", + "block_num": 119, + "block_time": "2020-09-06T02:00:47.000", + "producer_block_id": null, + "account_ram_deltas": [], + "account_disk_deltas": [], + "except": null, + "error_code": null, + "return_value_hex_data": "" + }, + { + "action_ordinal": 2, + "creator_action_ordinal": 0, + "closest_unnotified_ancestor_action_ordinal": 0, + "receipt": { + "receiver": "payloadless", + "act_digest": "b8871e8f3c79b02804a2ad28acb015f503e7f6e56f35565e5fa37b6767da1aa5", + "global_sequence": 157, + "recv_sequence": 3, + "auth_sequence": [ + [ + "payloadless", + 3 + ] + ], + "code_sequence": 1, + "abi_sequence": 1 + }, + "receiver": "payloadless", + "act": { + "account": "payloadless", + "name": "doit", + "authorization": [ + { + "actor": "payloadless", + "permission": "active" + } + ], + "data": "" + }, + "context_free": false, + "elapsed": 11, + "console": "Im a payloadless action", + "trx_id": "1b9a9c53f9b692d3382bcc19c0c21eb22207e2f51a30fe88dabbb45376b6ff23", + "block_num": 119, + "block_time": "2020-09-06T02:00:47.000", + "producer_block_id": null, + "account_ram_deltas": [], + "account_disk_deltas": [], + "except": null, + "error_code": null, + "return_value_hex_data": "" + } + ] +} +``` + +### Remarks + +Notice the following modified fields within the pruned transaction: + * The `prunable_data["prunable_data"][0]` field is 1. + * The `signatures` field now contains an empty array. + * The `context_free_data` field contains an empty array. + * The `packed_context_free_data` field is removed. + +## Next Steps + +The following actions are available after you complete the procedure: + +* Verify that the pruned transaction indeed contains pruned context-free data. +* Display the block that contains the pruned transaction from the block logs. diff --git a/docs/01_nodeos/07_concepts/10_context-free-data/index.md b/docs/01_nodeos/07_concepts/10_context-free-data/index.md index 0a31072bcbe..01565e1e13d 100644 --- a/docs/01_nodeos/07_concepts/10_context-free-data/index.md +++ b/docs/01_nodeos/07_concepts/10_context-free-data/index.md @@ -6,6 +6,9 @@ link_text: Context-Free Data ## Overview The immutable nature of the blockchain allows data to be stored securely while also enforcing the integrity of such data. However, this benefit also complicates the removal of non-essential data from the blockchain. Consequently, EOSIO blockchains contain a special section within the transaction, called the *context-free data*. As its name implies, data stored in the context-free data section is considered free of previous contexts or dependencies, which makes their potential removal possible. More importantly, such removal can be performed safely without compromising the integrity of the blockchain. +[[info | Blockchain Integrity]] +| Pruning of context-free data does not bend or relax the security of the blockchain. Nodes configured in full validation mode can still detect integrity violations on blocks with pruned transaction data. + ## Concept The goal of context-free data is to allow blockchain applications the option to store non-essential information within a transaction. Some examples of context-free data include: @@ -14,3 +17,27 @@ The goal of context-free data is to allow blockchain applications the option to * User comments made to an online article stored on the blockchain In general, any data that is not vital for the operation and integrity of the blockchain may be stored as context-free data. It may also be used to comply with regional laws and regulations concerning data usage and personal information. + +## Pruning +Blockchain applications that use context-free data might also want to remove the contents without affecting blockchain integrity. This can be achieved through a process called *pruning*. Removal of the context-free data associated with a transaction provides more functionality to blockchain applications, namely: + +* A mechanism to delete transaction data free of any context or inter-dependencies. +* A way to maintain blockchain integrity while removing such context-free data. + +Pruning of context-free data only allows light block validation between trusted nodes. Full block validation, which involves transaction signature verification and permission authorization checks, is not fully feasible without violating the integrity checks of blocks and transactions where the pruning occurred. + +[[info | Pruning on Private Blockchains]] +| Private EOSIO blockchains can benefit the most from context-free data pruning. Their controlled environment allows for trusted nodes to operate in light validation mode. This allows blockchain applications to use private EOSIO blockchains for this powerful feature. + +### Pruning Support +`nodeos` supports the pruning of context-free data by meeting the following requirements: + +* Correct handling of irreversible blocks with removed context-free data in pruned transactions +* Efficient deletion of existing context-free data within arbitrary finalized transactions +* Correct handling of transaction traces with removed context-free data generated by the state history plugin +* Efficient deletion of existing context-free data within finalized transactions from the trace log used by the state history plugin +* Peer-to-peer synchronization of blocks with removed context-free data from applicable transactions +* Tool support for the actual CFD pruning within the irreversible blocks log and the state history plugin trace log + +[[info | Pruning Tool]] +| Node operators can perform the pruning of context-free data within a given transaction by using the [`eosio-blocklog`](../../../10_utilities/eosio-blocklog.md) utility. For instructions on using this tool for CFD pruning, see [How to prune context-free data](05_how-to-prune-context-free-data.md). diff --git a/docs/01_nodeos/08_troubleshooting/index.md b/docs/01_nodeos/08_troubleshooting/index.md index bb4a0d7edcb..e02265fdde0 100644 --- a/docs/01_nodeos/08_troubleshooting/index.md +++ b/docs/01_nodeos/08_troubleshooting/index.md @@ -46,4 +46,4 @@ cleos --url http://localhost:8888 get info | grep server_version ### Error 3070000: WASM Exception Error -If you try to deploy the `eosio.bios` contract or `eosio.system` contract in an attempt to boot an EOSIO-based blockchain and you get the following error or similar: `Publishing contract... Error 3070000: WASM Exception Error Details: env.set_proposed_producers_ex unresolveable`, it is because you have to activate the `PREACTIVATE_FEATURE` protocol first. More details about it and how to enable it can be found in the [Bios Boot Sequence Tutorial](https://developers.eos.io/welcome/latest/tutorials/bios-boot-sequence/#112-set-the-eosiosystem-contract). For more information, you may also visit the [Nodeos Upgrade Guides](https://developers.eos.io/manuals/eos/latest/nodeos/upgrade-guides/). +If you try to deploy the `eosio.bios` contract or `eosio.system` contract in an attempt to boot an EOSIO-based blockchain and you get the following error or similar: `Publishing contract... Error 3070000: WASM Exception Error Details: env.set_proposed_producers_ex unresolveable`, it is because you have to activate the `PREACTIVATE_FEATURE` protocol first. More details about it and how to enable it can be found in the [Bios Boot Sequence Tutorial](https://developers.eos.io/welcome/v2.1/tutorials/bios-boot-sequence/#112-set-the-eosiosystem-contract). For more information, you may also visit the [Nodeos Upgrade Guides](https://developers.eos.io/manuals/eos/latest/nodeos/upgrade-guides/). diff --git a/docs/02_cleos/02_how-to-guides/how-to-buy-ram.md b/docs/02_cleos/02_how-to-guides/how-to-buy-ram.md index 6dc5f723b5e..7829866f4d8 100644 --- a/docs/02_cleos/02_how-to-guides/how-to-buy-ram.md +++ b/docs/02_cleos/02_how-to-guides/how-to-buy-ram.md @@ -1,23 +1,52 @@ -## Goal -Acquire RAM for contract deployment, database tables, and other blockchain resources +## Overview +This guide provides instructions on how to buy RAM for an EOSIO blockchain account using the cleos CLI tool. RAM is a system resource used to store blockchain state such as smart contract data and account information. -## Before you begin +The example uses `cleos` to buy RAM for the alice account. The alice account pays for the RAM and the alice@active permisssion authorizes the transaction. -* You have an account +## Before you Begin +Make sure you meet the following requirements: -* Ensure the reference system contracts from `eosio.contracts` repository is deployed and used to manage system resources +* Install the currently supported version of `cleos.` +[[info | Note]] +| `Cleos` is bundled with the EOSIO software. [Installing EOSIO](../../00_install/index.md) will install the `cleos` and `keosd` command line tools. +* You have access to an EOSIO blockchain and the `eosio.system` reference contract from [`eosio.contracts`](https://github.com/EOSIO/eosio.contracts) repository is deployed and used to manage system resources. +* You have an EOSIO account and access to the account's private key. +* You have sufficient [tokens allocated](how-to-transfer-an-eosio.token-token.md) to your account. -* You have sufficient token allocated to your account +## Reference +See the following reference guides for command line usage and related options: -* Install the currently supported version of `cleos` +* [cleos system buyram](../03_command-reference/system/system-buyram.md) command -* Unlock your wallet +## Buy Ram Procedure -## Steps +The following step shows you how to buy RAM -Buys RAM in value of 0.1 SYS tokens for account `alice`: +1. Run the following command to buy RAM worth 0.1 of `SYS` tokens for the alice account: -```sh +```shell cleos system buyram alice alice "0.1 SYS" -p alice@active -``` \ No newline at end of file +``` + +**Where** +* `alice` = payer, the account paying for RAM. +* `alice` = receiver, the account receiving bought RAM. +* `0.1 SYS` = The amount of tokens to pay for RAM. +* `-p alice@active` = The permission used to authorize the payment, in this case the active permission. + +**Example Output** + +```sh +executed transaction: aa243c30571a5ecc8458cb971fa366e763682d89b636fe9dbe7d28327d1cc4e9 128 bytes 283 us +# eosio <= eosio::buyram {"payer":"alice","receiver":"alice","quant":"0.1000 SYS"} +# eosio.token <= eosio.token::transfer {"from":"alice","to":"eosio.ram","quantity":"0.0995 SYS","memo":"buy ram"} +# eosio.token <= eosio.token::transfer {"from":"alice","to":"eosio.ramfee","quantity":"0.0005 SYS","memo":"ram fee"} +# alice <= eosio.token::transfer {"from":"alice","to":"eosio.ram","quantity":"0.0995 SYS","memo":"buy ram"} +# eosio.ram <= eosio.token::transfer {"from":"alice","to":"eosio.ram","quantity":"0.0995 SYS","memo":"buy ram"} +# alice <= eosio.token::transfer {"from":"alice","to":"eosio.ramfee","quantity":"0.0005 SYS","memo":"ram fee"} +# eosio.ramfee <= eosio.token::transfer {"from":"alice","to":"eosio.ramfee","quantity":"0.0005 SYS","memo":"ram fee"} +warning: transaction executed locally, but may not be confirmed by the network yet ] +``` +## Summary +In conclusion, by following these instructions you are able to purchase RAM, with a specified amount of tokens, for the specified accounts. \ No newline at end of file diff --git a/docs/02_cleos/02_how-to-guides/how-to-connect-a-specific-network.md b/docs/02_cleos/02_how-to-guides/how-to-connect-a-specific-network.md deleted file mode 100644 index 3b4698c2ba3..00000000000 --- a/docs/02_cleos/02_how-to-guides/how-to-connect-a-specific-network.md +++ /dev/null @@ -1,25 +0,0 @@ -## Goal - -Connect to a specific `nodeos` or `keosd` host to send COMMAND - -`cleos` and `keosd` can connect to a specific node by using the `--url` or `--wallet-url` optional arguments, respectively, followed by the http address and port number these services are listening to. - -[[info | Default address:port]] -| If no optional arguments are used (i.e. `--url` or `--wallet-url`), `cleos` attempts to connect to a local `nodeos` or `keosd` running at localhost `127.0.0.1` and default port `8888`. - -## Before you begin - -* Install the currently supported version of `cleos` - -## Steps -### Connecting to Nodeos - -```sh -cleos -url http://nodeos-host:8888 COMMAND -``` - -### Connecting to Keosd - -```sh -cleos --wallet-url http://keosd-host:8888 COMMAND -``` diff --git a/docs/02_cleos/02_how-to-guides/how-to-connect-to-a-specific-keosd.md b/docs/02_cleos/02_how-to-guides/how-to-connect-to-a-specific-keosd.md new file mode 100644 index 00000000000..a5e3dfefa16 --- /dev/null +++ b/docs/02_cleos/02_how-to-guides/how-to-connect-to-a-specific-keosd.md @@ -0,0 +1,35 @@ +## Overview +This guide provides instructions on how to connect to specifc wallet when using `cleos`. `Cleos` can connect to a specific wallet by using the `--wallet-url` optional argument, followed by the http address and port number. + +The example uses the `wallet-url` optional arguments to request data from the the specified `keosd` instance. + +[[info | Default address:port]] +| If no optional arguments are used (i.e. no `--wallet-url`), `cleos` attempts to connect to a local `nodeos` or `keosd` running at localhost or `127.0.0.1` and default port `8900`. Use the `keosd` command line arguments or [config.ini](../../03_keosd/10_usage.md/#launching-keosd-manually) file to specify a different address. + +## Before you Begin +Make sure you meet the following requirements: + +* Install the currently supported version of `cleos` and `keosd`. +[[info | Note]] +| The `cleos` tool and `keosd` are bundled with the EOSIO software. [Installing EOSIO](../../00_install/index.md) will install the `cleos` and `keosd` command line tools. +* You have access to an EOSIO blockchain and the http address and port number of a `nodeos` instance. + +## Reference +See the following reference guides for command line usage and related options: + +* [cleos](../index.md) command + +## Example + +1. Add the `--wallet-url` option to specify the `keosd` instance + +```shell +cleos --wallet-url http://keosd-host:8900 COMMAND +``` + +**where** +* `--wallet-url http://keosd-host:8900` = The http address and port number of the `keosd` instance to connect to +* COMMAND = The `cleos`command. + +## Summary +In conclusion, by following these instructions you are able to connect to a specified `keosd` instance. diff --git a/docs/02_cleos/02_how-to-guides/how-to-connect-to-a-specific-network.md b/docs/02_cleos/02_how-to-guides/how-to-connect-to-a-specific-network.md new file mode 100644 index 00000000000..65d155d05c1 --- /dev/null +++ b/docs/02_cleos/02_how-to-guides/how-to-connect-to-a-specific-network.md @@ -0,0 +1,34 @@ +## Overview +This guide provides instructions on how to connect to specifc EOSIO blockchain when using `cleos`. `Cleos` can connect to a specific node by using the `--url` optional argument, followed by the http address and port number. + +The examples use the `--url`optional argument to send commands to the specified blockchain. + +[[info | Default address:port]] +| If no optional arguments are used (i.e. no `--url`), `cleos` attempts to connect to a local `nodeos` running at localhost or `127.0.0.1` and default port `8888`. Use the `nodeos` command line arguments or [config.ini](../../01_nodeos/02_usage/01_nodeos-configuration/#configini-location) file to specify a different address. + +## Before you Begin +Make sure you meet the following requirements: + +* Install the currently supported version of `cleos`. +[[info | Note]] +| `Cleos` is bundled with the EOSIO software. [Installing EOSIO](../../00_install/index.md) will install the `cleos` and `keosd` command line tools. +* You have access to an EOSIO blockchain and the http afddress and port number of a `nodeos` instance. + +## Reference +See the following reference guides for command line usage and related options: + +* [cleos](../index.md) command + +## Example + +1. Add the `-url` option to specify the `nodeos` instance + +```shell +cleos -url http://nodeos-host:8888 COMMAND +``` +**Where** +* `-url http://nodeos-host:8888` = The http address and port number of the `nodeos` instance to connect to +* COMMAND = The `cleos`command. + +## Summary +In conclusion, by following these instructions you are able to connect to a specified `nodeos` instance. diff --git a/docs/02_cleos/02_how-to-guides/how-to-create-a-wallet.md b/docs/02_cleos/02_how-to-guides/how-to-create-a-wallet.md index 4973217f161..8d6b45eab7d 100644 --- a/docs/02_cleos/02_how-to-guides/how-to-create-a-wallet.md +++ b/docs/02_cleos/02_how-to-guides/how-to-create-a-wallet.md @@ -13,9 +13,9 @@ Make sure you meet the following requirements: [[info | Note]] | `cleos` is bundled with the EOSIO software. [Installing EOSIO](../../00_install/index.md) will also install `cleos`. -* Understand what an [account](https://developers.eos.io/welcome/latest/glossary/index/#account) is and its role in the blockchain. -* Understand [Accounts and Permissions](https://developers.eos.io/welcome/latest/protocol-guides/accounts_and_permissions) in the protocol documents. -* Understand what a [public](https://developers.eos.io/welcome/latest/glossary/index/#public-key) and [private](https://developers.eos.io/welcome/latest/glossary/index/#private-key) key pair is. +* Understand what an [account](https://developers.eos.io/welcome/v2.1/glossary/index/#account) is and its role in the blockchain. +* Understand [Accounts and Permissions](https://developers.eos.io/welcome/v2.1/protocol-guides/accounts_and_permissions) in the protocol documents. +* Understand what a [public](https://developers.eos.io/welcome/v2.1/glossary/index/#public-key) and [private](https://developers.eos.io/welcome/v2.1/glossary/index/#private-key) key pair is. ## Steps diff --git a/docs/02_cleos/02_how-to-guides/how-to-create-an-account.md b/docs/02_cleos/02_how-to-guides/how-to-create-an-account.md index 53908938c99..9ef26d31d5a 100644 --- a/docs/02_cleos/02_how-to-guides/how-to-create-an-account.md +++ b/docs/02_cleos/02_how-to-guides/how-to-create-an-account.md @@ -1,21 +1,47 @@ ## Goal +Create a new EOSIO blockchain account ## Before you begin * Install the currently supported version of `cleos` -* Understand the following: - * What is an account - * What is a public and private key pair +[[info | Note]] +| The cleos tool is bundled with the EOSIO software. [Installing EOSIO](../../00_install/index.md) will also install the cleos tool. + +* Acquire functional understanding of the following: + * [EOSIO Accounts and Permissions](https://developers.eos.io/welcome/v2.1/protocol/accounts_and_permissions) + * Asymmetric cryptography (public and private keypair) * Created an Owner and an Active key pair -* Imported a key pair which can authorize on behalf of a creator account +* [Import a private key](../02_how-to-guides/how-to-import-a-key.md) to the wallet which can authorize on behalf of a creator account ## Steps +**Command** + ```sh -cleos create account creator name OwnerKey [ActiveKey] +cleos create account [creator account name] [new account name][OwnerKey] [ActiveKey] ``` +Where: + +[creator account name] = name of the existing account that authorizes the creation of a new account + +[new account name] = The name of the new account account adhering to EOSIO account naming conventions + +[OwnerKey] = The owner permissions linked to the ownership of the account + +[ActiveKey] = The active permissions linked with the creator account [[info | Recommend]] | `ActiveKey` is optional but recommended. + +[[info | Note]] +| To create a new account in the EOSIO blockchain, an existing account, also referred to as a creator account, is required to authorize the creation of a new account. For a newly created EOSIO blockchain, the default system account used to create a new account is eosio. + +**Example Output** +```sh +cleos create account eosio bob EOS87TQktA5RVse2EguhztfQVEh6XXxBmgkU8b4Y5YnGvtYAoLGNN +executed transaction: 4d65a274de9f809f9926b74c3c54aadc0947020bcfb6dd96043d1bcd9c46604c 200 bytes 166 us +# eosio <= eosio::newaccount {"creator":"eosio","name":"bob","owner":{"threshold":1,"keys":[{"key":"EOS87TQktA5RVse2EguhztfQVEh6X... +warning: transaction executed locally, but may not be confirmed by the network yet ] +``` diff --git a/docs/02_cleos/02_how-to-guides/how-to-create-key-pairs.md b/docs/02_cleos/02_how-to-guides/how-to-create-key-pairs.md index dc0a77bca7c..9ebb6b35836 100644 --- a/docs/02_cleos/02_how-to-guides/how-to-create-key-pairs.md +++ b/docs/02_cleos/02_how-to-guides/how-to-create-key-pairs.md @@ -1,22 +1,50 @@ ## Goal +Create a keypair consisting of a public and a private key for signing transactions in the EOSIO blockchain. ## Before you begin +Before you follow the steps to create a new key pair, make sure the following items are fulfilled: + * Install the currently supported version of `cleos` -* Understand the following: - * What is a public and private key pair +[[info | Note]] +| The cleos tool is bundled with the EOSIO software. [Installing EOSIO](../../00_install/index.md) will also install the cleos tool. + +* Acquire functional understanding of asymmetric cryptography (public and private keypair) in the context of blockchain ## Steps -To output the key pair to the console +To create a keypair and print the result to the console: + +```sh +cleos create key --to-console +``` + +**Example Output** ```sh cleos create key --to-console +Private key: 5KPzrqNMJdr6AX6abKg*******************************cH +Public key: EOS4wSiQ2jbYGrqiiKCm8oWR88NYoqnmK4nNL1RCtSQeSFkGtqsNc ``` -To save the key pair to file + +To create a keypair and save it to a file: ```sh cleos create key --file FILE_TO_SAVEKEY -``` \ No newline at end of file +``` +Where: FILE_TO_SAVEKEY = name of the file + +**Example Output** +```sh +cleos create key --file pw.txt +saving keys to pw.txt +``` + +To view the saved keypair in the file: +```sh +cat pw.txt +Private key: 5K7************************************************ +Public key: EOS71k3WdpLDeqeyqVRAAxwpz6TqXwDo9Brik5dQhdvvpeTKdNT59 +``` diff --git a/docs/02_cleos/02_how-to-guides/how-to-delegate-CPU-resource.md b/docs/02_cleos/02_how-to-guides/how-to-delegate-CPU-resource.md index 5345bd83f63..c5e5b31aa61 100644 --- a/docs/02_cleos/02_how-to-guides/how-to-delegate-CPU-resource.md +++ b/docs/02_cleos/02_how-to-guides/how-to-delegate-CPU-resource.md @@ -13,9 +13,9 @@ Make sure you meet the following requirements: | `cleos` is bundled with the EOSIO software. [Installing EOSIO](../../00_install/index.md) will also install `cleos`. * Ensure the reference system contracts from [`eosio.contracts`](https://github.com/EOSIO/eosio.contracts) repository is deployed and used to manage system resources. -* Understand what an [account](https://developers.eos.io/welcome/latest/glossary/index/#account) is and its role in the blockchain. -* Understand [CPU bandwidth](https://developers.eos.io/welcome/latest/glossary/index/#cpu) in an EOSIO blockchain. -* Understand [NET bandwidth](https://developers.eos.io/welcome/latest/glossary/index/#net) in an EOSIO blockchain. +* Understand what an [account](https://developers.eos.io/welcome/v2.1/glossary/index/#account) is and its role in the blockchain. +* Understand [CPU bandwidth](https://developers.eos.io/welcome/v2.1/glossary/index/#cpu) in an EOSIO blockchain. +* Understand [NET bandwidth](https://developers.eos.io/welcome/v2.1/glossary/index/#net) in an EOSIO blockchain. ## Steps diff --git a/docs/02_cleos/02_how-to-guides/how-to-delegate-net-resource.md b/docs/02_cleos/02_how-to-guides/how-to-delegate-net-resource.md index 118c36ba13a..8de80eeb74d 100644 --- a/docs/02_cleos/02_how-to-guides/how-to-delegate-net-resource.md +++ b/docs/02_cleos/02_how-to-guides/how-to-delegate-net-resource.md @@ -13,9 +13,9 @@ Make sure you meet the following requirements: | `cleos` is bundled with the EOSIO software. [Installing EOSIO](../../00_install/index.md) will also install `cleos`. * Ensure the reference system contracts from [`eosio.contracts`](https://github.com/EOSIO/eosio.contracts) repository is deployed and used to manage system resources. -* Understand what an [account](https://developers.eos.io/welcome/latest/glossary/index/#account) is and its role in the blockchain. -* Understand [NET bandwidth](https://developers.eos.io/welcome/latest/glossary/index/#net) in an EOSIO blockchain. -* Understand [CPU bandwidth](https://developers.eos.io/welcome/latest/glossary/index/#cpu) in an EOSIO blockchain. +* Understand what an [account](https://developers.eos.io/welcome/v2.1/glossary/index/#account) is and its role in the blockchain. +* Understand [NET bandwidth](https://developers.eos.io/welcome/v2.1/glossary/index/#net) in an EOSIO blockchain. +* Understand [CPU bandwidth](https://developers.eos.io/welcome/v2.1/glossary/index/#cpu) in an EOSIO blockchain. ## Steps diff --git a/docs/02_cleos/02_how-to-guides/how-to-get-account-information.md b/docs/02_cleos/02_how-to-guides/how-to-get-account-information.md index c3dfd3b3bb2..b294afbea61 100644 --- a/docs/02_cleos/02_how-to-guides/how-to-get-account-information.md +++ b/docs/02_cleos/02_how-to-guides/how-to-get-account-information.md @@ -6,8 +6,11 @@ Query infomation of an EOSIO account * Install the currently supported version of `cleos` -* Understand the following: - * What is an account +[[info | Note]] +| The cleos tool is bundled with the EOSIO software. [Installing EOSIO](../../00_install/index.md) will also install the cleos tool. + +* Acquire functional understanding of [EOSIO Accounts and Permissions](https://developers.eos.io/welcome/v2.1/protocol/accounts_and_permissions) + ## Steps @@ -16,8 +19,9 @@ Execute the command below: ```sh cleos get account ACCOUNT_NAME ``` +Where ACCOUNT_NAME = name of the existing account in the EOSIO blockchain. -This should produce similar output as below: +**Example Output** ```console created: 2018-06-01T12:00:00.000 diff --git a/docs/02_cleos/02_how-to-guides/how-to-get-block-information.md b/docs/02_cleos/02_how-to-guides/how-to-get-block-information.md index 7efd62427f6..b35ccf12e4f 100644 --- a/docs/02_cleos/02_how-to-guides/how-to-get-block-information.md +++ b/docs/02_cleos/02_how-to-guides/how-to-get-block-information.md @@ -12,20 +12,20 @@ Make sure to meet the following requirements: [[info | Note]] | `cleos` is bundled with the EOSIO software. [Installing EOSIO](../../00_install/index.md) will also install `cleos`. -* Understand what a [block](https://developers.eos.io/welcome/latest/glossary/index/#block) is and its role in the blockchain. -* Understand the [block lifecycle](https://developers.eos.io/welcome/latest/protocol-guides/consensus_protocol/#5-block-lifecycle) in the EOSIO consensus protocol. +* Understand what a [block](https://developers.eos.io/welcome/v2.1/glossary/index/#block) is and its role in the blockchain. +* Understand the [block lifecycle](https://developers.eos.io/welcome/v2.1/protocol-guides/consensus_protocol/#5-block-lifecycle) in the EOSIO consensus protocol. ## Steps Perform the step below: -Retrieve full information about a block: +Retrieve full or partial information about a block: ```sh -cleos get block +cleos get block [--info] ``` -Where `block_number_or_id` is the specified block number or block ID. +Where `block_number_or_id` is the specified block number or block ID and `--info` is an optional parameter to retrieve a partial subset of the block information. Some examples are provided below: @@ -78,3 +78,29 @@ cleos -u https://api.testnet.eos.io get block 02e1c7888a92206573ae38d00e09366c7b "ref_block_prefix": 3493375603 } ``` + +* Query the local chain to retrieve partial block information about block number `1`: + +**Example Output** + +```sh +cleos get block --info 1 +``` +```json +{ + "block_num": 1, + "ref_block_num": 1, + "id": "0000000130d70e94e0022fd2fa035cabb9e542c34ea27f572ac90b5a7aa3d891", + "timestamp": "2018-03-02T12:00:00.000", + "producer": "", + "confirmed": 1, + "previous": "0000000000000000000000000000000000000000000000000000000000000000", + "transaction_mroot": "0000000000000000000000000000000000000000000000000000000000000000", + "action_mroot": "0000000000000000000000000000000000000000000000000000000000000000", + "schedule_version": 0, + "producer_signature": "SIG_K1_111111111111111111111111111111111111111111111111111111111111111116uk5ne", + "ref_block_prefix": 3526296288 +} +``` + +Note that the partial block information excludes the variable fields `new_producers`, `header_extensions`, `transactions`, or `block_extensions`. diff --git a/docs/02_cleos/02_how-to-guides/how-to-link-permission.md b/docs/02_cleos/02_how-to-guides/how-to-link-permission.md index 7fec13730bd..0ab8da650de 100644 --- a/docs/02_cleos/02_how-to-guides/how-to-link-permission.md +++ b/docs/02_cleos/02_how-to-guides/how-to-link-permission.md @@ -1,20 +1,43 @@ -## Goal +## Overview +This guide provides instructions to link a permission to a smart contract action. -Link a permission to an action of a contract +The example uses `cleos` to link a custom permission _customp_ in the account _alice_ to a _hi_ action deployed to the _scontract_ account so that the _alice_ account's `active` permission and _customp_ permission are authorized to call the _hi_ _action. -## Before you begin +## Before you Begin +Make sure you meet the following requirements: -* Install the currently supported version of `cleos` +* Install the currently supported version of `cleos.` +[[info | Note]] +| `Cleos` is bundled with the EOSIO software. [Installing EOSIO](../../00_install/index.md) will also install the `cleos` and `keosd` comand line tools. +* You have an EOSIO account and access to the account's `active` private key. +* You have created a custom permission. See [cleos set account permission](../03_command-reference/set/set-account-permission.md). -* Understand the following: - * What is an account - * What is permission level - * What is an action +## Command Reference +See the following reference guides for command line usage and related options: -## Steps +* [cleos set action permission](../03_command-reference/set/set-action-permission.md) command +## link Procedure -Link a permission level `permlvl` to the action `transfer` of contract `hodlcontract` +The following step shows you how to link a permission: + +1. Run the following command to link _alices_ account permission _customp_ with the _hi_ action deployed to the _scontract_ account: + +```shell +cleos set action permission alice scontract hi customp -p alice@active +``` + +**Where** +* `alice` = The name of the account containing the permission to link. +* `scontract`= The name of the account which owns the smart contract. +* `hi` = The name of the action to link to a permission. +* `customp` = The permission used to authorize the transaction. +* `-p alice@active` = The permission used to authorize linking the _customp_ permission. + +**Example Output** +```shell +executed transaction: 4eb4cf3aea232d46e0e949bc273c3f0575be5bdba7b61851ab51d927cf74a838 128 bytes 141 us +# eosio <= eosio::linkauth {"account":"alice","code":"scontract","type":"hi","requirement":"customp"} +``` +## Summary +In conclusion, by following these instructions you are able to link a permission to a smart contract action. -```sh -cleos set action permission alice hodlcontract transfer permlvl -``` \ No newline at end of file diff --git a/docs/02_cleos/02_how-to-guides/how-to-stake-resource.md b/docs/02_cleos/02_how-to-guides/how-to-stake-resource.md index 6a9566f424d..9b243067d4f 100644 --- a/docs/02_cleos/02_how-to-guides/how-to-stake-resource.md +++ b/docs/02_cleos/02_how-to-guides/how-to-stake-resource.md @@ -1,28 +1,111 @@ -## Goal +## Overview -Stake resource for your account +This how-to guide provides instructions on how to stake resources, NET and/or CPU, for your account using the `cleos` CLI tool. ## Before you begin -* Install the currently supported version of `cleos` +* Install the currently supported version of `cleos`. -* Ensure the reference system contracts from `eosio.contracts` repository is deployed and used to manage system resources +* Ensure the [reference system contracts](https://developers.eos.io/manuals/eosio.contracts/v1.9/build-and-deploy) are deployed and used to manage system resources. * Understand the following: - * What is an account - * What is network bandwidth - * What is CPU bandwidth + * What an [account](https://developers.eos.io/welcome/v2.1/glossary/index/#account) is. + * What [NET bandwidth](https://developers.eos.io/manuals/eosio.contracts/v1.9/key-concepts/net) is. + * What [CPU bandwidth](https://developers.eos.io/manuals/eosio.contracts/v1.9/key-concepts/cpu) is. + * The [`delegatebw` cleos sub-command](https://developers.eos.io/manuals/eos/v2.1/cleos/command-reference/system/system-delegatebw). -## Steps +## Command Reference -Stake 0.01 SYS network bandwidth for `alice` +See the following reference guides for command line usage and related options for the `cleos` command: + +* The [`delegatebw` cleos sub-command](https://developers.eos.io/manuals/eos/v2.1/cleos/command-reference/system/system-delegatebw). + +## Procedure + +The following steps show: + +1. [How to stake NET bandwidth.](#1-stake-net-bandwidth) +2. [How to stake CPU bandwidth.](#2-stake-cpu-bandwidth) +3. [How to stake NET and CPU bandwidth.](#3-stake-net-and-cpu-bandwidth) + +### 1. Stake NET bandwidth + +Run the following command to stake `0.01 SYS` of NET bandwidth for `alice` account from `bob` account: ```sh -cleos system delegatebw alice alice "0 SYS" "0.01 SYS" +cleos system delegatebw alice bob "0.01 SYS" "0 SYS" +``` + +Where: + +* `alice` = the account for which the NET bandwidth is staked. +* `bob` = the account that pays the `0.01 SYS` for the NET bandwidth staked. +* `0.01 SYS` = the amount of `SYS` tokens allocated to stake NET bandwidth. +* `0 SYS` = the amount of `SYS` tokens allocated to stake CPU bandwidth. + +Example output: + +```console +executed transaction: 5487afafd67bf459a20fcc2dbc5d0c2f0d1f10e33123eaaa07088046fd18e3ae 192 bytes 503 us +# eosio <= eosio::delegatebw {"from":"bob","receiver":"alice","stake_net_quantity":"0.01 SYS","stake_cpu_quanti... +# eosio.token <= eosio.token::transfer {"from":"bob","to":"eosio.stake","quantity":"0.01 EOS","memo":"stake bandwidth"} +# alice <= eosio.token::transfer {"from":"bob","to":"eosio.stake","quantity":"0.01 SYS","memo":"stake bandwidth"} +# eosio.stake <= eosio.token::transfer {"from":"bob","to":"eosio.stake","quantity":"0.01 SYS","memo":"stake bandwidth"} ``` -Stake 0.01 SYS CPU bandwidth for `alice`: +### 2. Stake CPU bandwidth + +Run the following command to stake `0.01 SYS` of CPU bandwidth for `alice` account from `bob` account: ```sh -cleos system delegatebw alice alice "0.01 SYS" "0 SYS" -``` \ No newline at end of file +cleos system delegatebw alice bob "0 SYS" "0.01 SYS" +``` + +Where: + +* `alice` = the account for which the CPU bandwidth is staked. +* `bob` = the account that pays the `0.01 SYS` for the CPU bandwidth staked. +* `0 SYS` = the amount of `SYS` tokens allocated to stake NET bandwidth. +* `0.01 SYS` = the amount `SYS` tokens allocated to stake CPU bandwidth. + +Example output: + +```console +executed transaction: 5487afafd67bf459a20fcc2dbc5d0c2f0d1f10e33123eaaa07088046fd18e3ae 192 bytes 503 us +# eosio <= eosio::delegatebw {"from":"bob","receiver":"alice","stake_net_quantity":"0.0000 SYS","stake_cpu_quanti... +# eosio.token <= eosio.token::transfer {"from":"bob","to":"eosio.stake","quantity":"0.01 EOS","memo":"stake bandwidth"} +# alice <= eosio.token::transfer {"from":"bob","to":"eosio.stake","quantity":"0.01 SYS","memo":"stake bandwidth"} +# eosio.stake <= eosio.token::transfer {"from":"bob","to":"eosio.stake","quantity":"0.01 SYS","memo":"stake bandwidth"} +``` + +### 3. Stake NET and CPU bandwidth + +Run the following command to stake `0.01 SYS` of NET and `0.02 SYS` of CPU bandwidth for `alice` account from `bob` account: + +```sh +cleos system delegatebw alice bob "0.01 SYS" "0.02 SYS" +``` + +Where: + +* `alice` = the account for which the NET and CPU bandwidth is staked. +* `bob` = the account that pays `0.01 SYS` for the NET and `0.02 SYS` for the CPU bandwidth staked. +* `0.01 SYS` = the amount of `SYS` tokens allocated to stake NET bandwidth. +* `0.02 SYS` = the amount of `SYS` tokens allocated to stake CPU bandwidth. + +Example output: + +```console +executed transaction: 5487afafd67bf459a20fcc2dbc5d0c2f0d1f10e33123eaaa07088046fd18e3ae 192 bytes 503 us +# eosio <= eosio::delegatebw {"from":"bob","receiver":"alice","stake_net_quantity":"0.01 SYS","stake_cpu_quanti... +# eosio.token <= eosio.token::transfer {"from":"bob","to":"eosio.stake","quantity":"0.01 EOS","memo":"stake bandwidth"} +# alice <= eosio.token::transfer {"from":"bob","to":"eosio.stake","quantity":"0.01 SYS","memo":"stake bandwidth"} +# eosio.stake <= eosio.token::transfer {"from":"bob","to":"eosio.stake","quantity":"0.01 SYS","memo":"stake bandwidth"} +``` + +[[info|An account can stake to itself]] +| An account can stake resources to itself, that is, `bob` account can be substituted in the above examples with `alice`, provided `alice` account holds sufficient `SYS` tokens. That means `alice` account stakes resources to itself. + +## Summary + +In conclusion, the above instructions show how to stake CPU and/or NET bandwidth from one account to another or to itself. diff --git a/docs/02_cleos/02_how-to-guides/how-to-submit-a-transaction.md b/docs/02_cleos/02_how-to-guides/how-to-submit-a-transaction.md index 89207f1cfe0..ded2288f30a 100644 --- a/docs/02_cleos/02_how-to-guides/how-to-submit-a-transaction.md +++ b/docs/02_cleos/02_how-to-guides/how-to-submit-a-transaction.md @@ -1,18 +1,35 @@ -## Goal +## Overview -Push a transaction +This how-to guide provides instructions on how to submit, or push, a transaction using the `cleos` CLI tool. ## Before you begin * Install the currently supported version of `cleos` * Understand the following: - * What is a transaction - * How to generate a valid transaction JSON + * What a [transaction](https://developers.eos.io/welcome/latest/glossary/index/#transaction) is. + * How to generate a valid transaction JSON. + * Consult [cleos push transaction](https://developers.eos.io/manuals/eos/v2.1/cleos/command-reference/push/push-transaction) reference, and pay attention to option `-d` and `-j`. + * Consult [push transaction](https://developers.eos.io/manuals/eos/v2.1/nodeos/plugins/chain_api_plugin/api-reference/index#operation/push_transaction) endpoint for chain api plug-in, and pay attention to the payload definition. -## Steps +## Command Reference -* Create a JSON snippet contains a valid transaction such as the following: +See the following reference guides for command line usage and related options for the `cleos` command: + +* The [cleos push transaction](https://developers.eos.io/manuals/eos/v2.1/cleos/command-reference/push/push-transaction) reference. + +## Procedure + +The following steps show how to: + +1. [Create the transaction as JSON snippet.](#1-create-the-transaction-as-json-snippet) +2. [Send the transaction.](#2-send-the-transaction) + +### 1. Create the transaction as JSON snippet + +You can create the JSON snippet defining the transaction in two ways: + +* Create a JSON snippet which contains a valid transaction and uses hex format for `data` field. ```JSON { @@ -39,12 +56,12 @@ Push a transaction } ``` -* You can also create a JSON snippet that uses clear text JSON for `data` field. +* Alternatively, you can also create a JSON snippet that uses clear text JSON for `data` field. [[info]] -| Be aware that if a clear text `data` field is used, cleos need to fetch copies of required ABIs using `nodeos` API. That operation has a performance overhead on `nodeos` +| Be aware that if a clear text `data` field is used, `cleos` needs to fetch the smart contract's ABI using `nodeos` API. This operation has an overall performance overhead for both `cleos` and `nodeos`. On the other hand if hex data is used in the `data` field then the ABI fetching is not executed and thus the total time to send and execute the transaction is faster. -```JSON +```json { "expiration": "2019-08-01T07:15:49", "ref_block_num": 34881, @@ -74,18 +91,22 @@ Push a transaction } ``` -* Execute the following command: +### 2. Send the transaction + +You can send the transaction you created as JSON snippet in two ways: + +* You can send the transaction stored in `TRX_FILE.json` file: ```sh cleos push transaction TRX_FILE.json ``` -* Submit a transaction from a JSON: +* Alternatively, you can send the transaction using the JSON content directly: ```sh -cleos push transaction JSON +cleos push transaction '{"expiration": "2019-08-01T07:15:49", "ref_block_num": 34881,"ref_block_prefix": 2972818865,"max_net_usage_words": 0,"max_cpu_usage_ms": 0,"delay_sec": 0,"context_free_actions": [],"actions": [{"account": "eosio.token","name": "transfer","authorization": [{"actor": "han","permission": "active"}],"data": {"from": "han","to": "eosio","quantity": "0.0001 SYS","memo": "m"}}],"transaction_extensions": [],"context_free_data": []}' ``` - \ No newline at end of file +## Summary + +In conclusion, the above instructions show how to submit, or push, a transaction using the `cleos` CLI tool. diff --git a/docs/02_cleos/02_how-to-guides/how-to-transfer-an-eosio.token-token.md b/docs/02_cleos/02_how-to-guides/how-to-transfer-an-eosio.token-token.md index 3abcb7f6916..71490662028 100644 --- a/docs/02_cleos/02_how-to-guides/how-to-transfer-an-eosio.token-token.md +++ b/docs/02_cleos/02_how-to-guides/how-to-transfer-an-eosio.token-token.md @@ -1,21 +1,47 @@ -## Goal +## Overview -Transfer token created by eosio.token contract +This how-to guide provides instructions on how to transfer tokens created by `eosio.token` contract. ## Before you begin -* Install the currently supported version of `cleos` +* Install the currently supported version of `cleos`. -* You are going to transfer a token created by eosio.token contract and eosio.token contract has been deployed on the network which you are connected to +* `eosio.token` contract is deployed on the network you are connected to. * Understand the following: - * What is a transaction - * Token transfers are irrevertable + * What a [transaction](https://developers.eos.io/welcome/v2.1/glossary/index/#transaction) is. + * Token transfers are irreversible. -## Steps +## Command Reference -Assume you would like to transfer `0.0001 SYS` token to an account called `bob` from an account called `alice`, execute the following: +See the following reference guides for command line usage and related options for the `cleos` command: + +* The [cleos transfer](https://developers.eos.io/manuals/eos/latest/cleos/command-reference/transfer) reference. + +## Procedure + +The following steps show how to transfer `0.0001 SYS` tokens to an account called `bob` from an account called `alice`: ```sh cleos transfer alice bob "0.0001 SYS" "Hodl!" -p alice@active ``` + +Where: + +* `alice` = the account that transfers the tokens. +* `bob` = the account that receives the tokens. +* `0.0001 SYS` = the amount of `SYS` tokens sent. +* `Hodl!` = the message, or memo, that is accompanying the transaction. + +Example output: + +```console +executed transaction: 800835f28659d405748f4ac0ec9e327335eae579a0d8e8ef6330e78c9ee1b67c 128 bytes 1073 us +# eosio.token <= eosio.token::transfer {"from":"alice","to":"bob","quantity":"25.0000 SYS","memo":"m"} +# alice <= eosio.token::transfer {"from":"alice","to":"bob","quantity":"25.0000 SYS","memo":"m"} +# bob <= eosio.token::transfer {"from":"alice","to":"bob","quantity":"25.0000 SYS","memo":"m"} +``` + +## Summary + +In conclusion, the above instructions show how to transfer tokens created by `eosio.token` contract from one account to another. diff --git a/docs/02_cleos/02_how-to-guides/how-to-update-account-keys.md b/docs/02_cleos/02_how-to-guides/how-to-update-account-keys.md new file mode 100644 index 00000000000..1295d379a94 --- /dev/null +++ b/docs/02_cleos/02_how-to-guides/how-to-update-account-keys.md @@ -0,0 +1,93 @@ + +## Overview +This how-to guide provides instructions on how to update an account keys for an EOSIO blockchain account using the cleos CLI tool. + +The example uses `cleos` to update the keys for the **alice** account. + +## Before you Begin +Make sure you meet the following requirements: + +* Install the currently supported version of `cleos.` +[[info | Note]] +| The `cleos` tool is bundled with the EOSIO software. [Installing EOSIO](../../00_install/index.md) will install the `cleos` and `keosd` command line tools. +* You have an EOSIO account and access to the account's private key. + +## Reference +See the following reference guides for command line usage and related options: + +* [cleos create key](../03_command-reference/create/key.md) command +* [cleos wallet import](../03_command-reference/wallet/import.md) command +* [cleos set account](../03_command-reference/set/set-account.md) command + +## Procedure +The following step shows how to change the keys for the `active` permissions: + +1. Create a new key pair for the `active` permission +```shell +cleos create key --to-console +``` +**Where** +`--to-console` = Tells the `cleos create key` command to print the private/public keys to the console. + +**Example Output** +```shell +Private key: 5KDNWQvY2seBPVUz7MiiaEDGTwACfuXu78bwZu7w2UDM9A3u3Fs +Public key: EOS5zG7PsdtzQ9achTdRtXwHieL7yyigBFiJDRAQonqBsfKyL3XhC +``` + +2. Import the new private key into your wallet +```shell +cleos wallet import --private-key 5KDNWQvY2seBPVUz7MiiaEDGTwACfuXu78bwZu7w2UDM9A3u3Fs +``` +**Where** +`--private-key 5KDNWQvY2seBPVUz7MiiaEDGTwACfuXu78bwZu7w2UDM9A3u3Fs` = The private key, in WIF format, to import. + +**Example Output** +```shell +imported private key for: EOS5zG7PsdtzQ9achTdRtXwHieL7yyigBFiJDRAQonqBsfKyL3XhC +``` + +3. Update the `active` permission key +```shell +cleos set account permission alice active EOS5zG7PsdtzQ9achTdRtXwHieL7yyigBFiJDRAQonqBsfKyL3XhC -p alice@owner +``` +**Where** +* `alice` = The name of the account to update the key. +* `active`= The name of the permission to update the key. +* `EOS5zG7PsdtzQ9achTdRtXwHieL7yyigBFiJDRAQonqBsfKyL3XhC` = The new public key. +* `-p alice@owner` = The permission used to authorize the transaction. + +**Example Output** +```shell +executed transaction: ab5752ecb017f166d56e7f4203ea02631e58f06f2e0b67103b71874f608793e3 160 bytes 231 us +# eosio <= eosio::updateauth {"account":"alice","permission":"active","parent":"owner","auth":{"threshold":1,"keys":[{"key":"E... +``` + +4. Check the account +```shell +cleos get account alice +``` +**Where** +`alice` = name, the name of the account to retrieve. + +**Example Output** +```shell +permissions: + owner 1: 1 EOS6c5UjmyRsZSdikLbpAoMdg4V7FQwvdhep3KMxUifzmpDnoLVPe + active 1: 1 EOS5zG7PsdtzQ9achTdRtXwHieL7yyigBFiJDRAQonqBsfKyL3XhC +memory: + quota: xxx used: 2.66 KiB + +net bandwidth: + used: xxx + available: xxx + limit: xxx + +cpu bandwidth: + used: xxx + available: xxx + limit: xxx +``` + +## Summary +In conclusion, by following these instructions you are able to change the keys used by an account. diff --git a/docs/02_cleos/02_how-to-guides/how-to-vote.md b/docs/02_cleos/02_how-to-guides/how-to-vote.md index b3eae710857..a5d43974158 100644 --- a/docs/02_cleos/02_how-to-guides/how-to-vote.md +++ b/docs/02_cleos/02_how-to-guides/how-to-vote.md @@ -1,30 +1,47 @@ -## Goal +## Overview -Vote for a block producer +This how-to guide provides instructions on how to vote for block producers. ## Before you begin -* Install the currently supported version of `cleos` +* Install the latest version of `cleos`. -* Ensure the reference system contracts from `eosio.contracts` repository is deployed and used to manage system resources +* Ensure the [reference system contracts](https://developers.eos.io/manuals/eosio.contracts/v1.9/build-and-deploy) are deployed and used to manage system resources. * Understand the following: - * What is a block producer - * How does voting works + * What a [block producer](https://developers.eos.io/welcome/v2.1/protocol-guides/consensus_protocol/#11-block-producers) is. + * How [voting](https://developers.eos.io/manuals/eosio.contracts/v1.9/key-concepts/vote) works. -* Unlock your wallet +* Unlock your wallet. -## Steps +## Command Reference -Assume you are going to vote for blockproducer1 and blockproducer2 from an account called `eosiotestts2`, execute the following: +See the following reference guides for command line usage and related options for the `cleos` command: + +* The [cleos system voteproducer prods](https://developers.eos.io/manuals/eos/v2.1/cleos/command-reference/system/system-voteproducer-prods) reference. + +## Procedure + +The following steps show: + +1. How to vote for blockproducer1 and blockproducer2 from an account called `eosiotestts2`: ```sh cleos system voteproducer prods eosiotestts2 blockproducer1 blockproducer2 ``` -This should produce similar output as below: +Where: + +* `eosiotestts2` = the account that votes. +* `blockproducer1` and `blockproducer2` = the accounts receiving the votes. The number of accounts receiving the votes can vary from one to multiple. Maximum default number of block producers one account can vote for is 30. + +Example output: ```console executed transaction: 2d8b58f7387aef52a1746d7a22d304bbbe0304481d7751fc4a50b619df62676d 128 bytes 374 us # eosio <= eosio::voteproducer {"voter":"eosiotestts2","proxy":"","producers":["blockproducer1","blockproducer2"]} ``` + +## Summary + +In conclusion, the above instructions show how to vote for block producers. diff --git a/docs/02_cleos/03_command-reference/create/key.md b/docs/02_cleos/03_command-reference/create/key.md index 81bce139358..7b875dfe0b6 100755 --- a/docs/02_cleos/03_command-reference/create/key.md +++ b/docs/02_cleos/03_command-reference/create/key.md @@ -1,28 +1,62 @@ -## Description +## Command +cleos create key [OPTIONS] -Creates a new keypair and prints the public and private keys +**Where** +* [OPTIONS] = See Options in Command Usage section below. You must choose one option of `--file` or `--to-console` -## Usage +**Note**: The arguments and options enclosed in square brackets are optional. -```console -Usage: cleos create key [OPTIONS] +## Description +Creates a new keypair and either prints the public and private keys to the screen or to a file. + +## Command Usage +The following information shows the different positionals and options you can use with the `cleos create key` command: + +### Positionals: +- none +### Options +- `-h,--help` - Print this help message and exit +- `--r1` - Generate a key using the R1 curve (iPhone), instead of the K1 curve (Bitcoin) +`-f`,`--file` _TEXT_ - Name of file to write private/public key output to. (Must be set, unless "--to-console" is passed +`--to-console` - Print private/public keys to console. + +## Requirements +* Install the currently supported version of `cleos`. +[[info | Note]] +| The `cleos` tool is bundled with the EOSIO software. [Installing EOSIO](../../00_install/index.md) will install the `cleos` and `keosd` command line tools. + +## Examples +1. Create a new key pair and output to the screen +```shell +cleos create key --to-console +``` +**Where** +`--to-console` = Tells the `cleos create key` command to print the private/public keys to the console. -Options: - -h,--help Print this help message and exit - --r1 Generate a key using the R1 curve (iPhone), instead of the K1 curve (Bitcoin) - -f,--file TEXT Name of file to write private/public key output to. (Must be set, unless "--to-console" is passed - --to-console Print private/public keys to console. +**Example Output** +```shell +Private key: 5KDNWQvY2seBPVUz7MiiaEDGTwACfuXu78bwZu7w2UDM9A3u3Fs +Public key: EOS5zG7PsdtzQ9achTdRtXwHieL7yyigBFiJDRAQonqBsfKyL3XhC ``` -## Command +2. Create a new key pair and output to a file +```shell +cleos create key --file my_keys.txt +``` +**Where** +`--file` keys.txt = Tells the `cleos create key` command to output the private/public keys to afile called `my_keys.txt`. -```sh -cleos create key -f passwd +**Example Output** +```shell +saving keys to my_keys.txt ``` -## Output +```shell +cat my_keys.txt +``` -```console -Private key: 5KCkcSxYKZfh5Cr8CCunS2PiUKzNZLhtfBjudaUnad3PDargFQo -Public key: EOS5uHeBsURAT6bBXNtvwKtWaiDSDJSdSmc96rHVws5M1qqVCkAm6 +**Example Output** +```shell +Private key: 5KWcfnGao5K6WV65Zgjd1xvpugRUeKwb6oxzmwaS1tPE2Ef4qzo +Public key: EOS7oDiBtzSHtShprGFmXm9JLnwxhcPeDyhww3tXDvVATDYtciLGG ``` diff --git a/docs/02_cleos/03_command-reference/get/account.md b/docs/02_cleos/03_command-reference/get/account.md index 8f38ebd6c93..d185cfc9522 100755 --- a/docs/02_cleos/03_command-reference/get/account.md +++ b/docs/02_cleos/03_command-reference/get/account.md @@ -1,20 +1,43 @@ + +## Command +cleos get account [OPTIONS] name [core-symbol] + +**Where** +* [OPTIONS] = See Options in Command Usage section below. + +**Note**: The arguments and options enclosed in square brackets are optional. + ## Description -Retrieves an account from the blockchain +Retrieve an account from the blockchain + +## Command Usage +The following information shows the different positionals and options you can use with the `cleos create key` command: -## Positional Parameters +### Positionals: - `name` _TEXT_ - The name of the account to retrieve -- `core-symbol` _TEXT_ - The expected core symbol of the chain you are querying +- `core-symbol` _TEXT_ - The expected core symbol of the chain you are querying - OPTIONAL -## Options +### Options +- `-h,--help` Print this help message and exit - `-j,--json` - Output in JSON format +## Requirements +* Install the currently supported version of `cleos.` +[[info | Note]] +| The `cleos` tool is bundled with the EOSIO software. [Installing EOSIO](../../00_install/index.md) will install the `cleos` and `keosd` command line tools. +* You have access to an EOSIO blockchain. + ## Examples -### Get formatted data for user **eosio** +1. Get formatted data for user **eosio** -```sh +```shell cleos get account eosio ``` +**Where** +`eosio` = The name of the account. + +**Example Output** ```console privileged: true permissions: @@ -35,11 +58,13 @@ cpu bandwidth: (averaged over 3 days) producers: ``` -### Get JSON data for user **eosio** +2. Get JSON data for user **eosio** ```sh cleos get account eosio --json ``` +**Example Output** + ```json { "account_name": "eosio", @@ -106,4 +131,4 @@ cleos get account eosio --json ``` ## See Also -- [Accounts and Permissions](https://developers.eos.io/welcome/latest/protocol/accounts_and_permissions) protocol document. +- [Accounts and Permissions](https://developers.eos.io/welcome/v2.1/protocol/accounts_and_permissions) protocol document. diff --git a/docs/02_cleos/03_command-reference/get/block.md b/docs/02_cleos/03_command-reference/get/block.md index 2d862827561..6952d4784bf 100755 --- a/docs/02_cleos/03_command-reference/get/block.md +++ b/docs/02_cleos/03_command-reference/get/block.md @@ -5,8 +5,10 @@ Retrieves a full block from the blockchain - `block` _TEXT_ - The number **or** ID of the block to retrieve ## Options - `--header-state` - Get block header state from fork database instead +- `--info` - Get block info from the blockchain by block num only ## Example +### Get the full block: ```sh cleos get block 1 @@ -37,3 +39,31 @@ This will output a block object similar to the following "ref_block_prefix": 3526296288 } ``` + +### Get the block info: + +```sh +cleos get block --info 1 +``` + +This will output a block info object similar to the following + +```json +{ + "block_num": 1, + "ref_block_num": 1, + "id": "0000000130d70e94e0022fd2fa035cabb9e542c34ea27f572ac90b5a7aa3d891", + "timestamp": "2018-03-02T12:00:00.000", + "producer": "", + "confirmed": 1, + "previous": "0000000000000000000000000000000000000000000000000000000000000000", + "transaction_mroot": "0000000000000000000000000000000000000000000000000000000000000000", + "action_mroot": "0000000000000000000000000000000000000000000000000000000000000000", + "schedule_version": 0, + "producer_signature": "SIG_K1_111111111111111111111111111111111111111111111111111111111111111116uk5ne", + "ref_block_prefix": 3526296288 +} +``` + +## Remarks +The block info object has a fixed size and excludes the following fields: new_producers, header_extensions, transactions, block_extensions. diff --git a/docs/02_cleos/03_command-reference/get/index.md b/docs/02_cleos/03_command-reference/get/index.md index 594852b921f..ca896f9fee2 100755 --- a/docs/02_cleos/03_command-reference/get/index.md +++ b/docs/02_cleos/03_command-reference/get/index.md @@ -9,6 +9,7 @@ Retrieves various items and information from the blockchain - [code](code.md) - Retrieve the code and ABI for an account - [abi](abi.md) - Retrieve the ABI for an account - [table](table.md) - Retrieve the contents of a database table +- [kv_table](kv_table.md) - Retrieve the contents of a database kv_table - [scope](scope.md) - Retrieve a list of scopes and tables owned by a contract - [currency](currency.md) - Retrieve information related to standard currencies - [accounts](accounts.md) - Retrieve accounts associated with a public key diff --git a/docs/02_cleos/03_command-reference/get/kv_table.md b/docs/02_cleos/03_command-reference/get/kv_table.md new file mode 100644 index 00000000000..b378e28c2d6 --- /dev/null +++ b/docs/02_cleos/03_command-reference/get/kv_table.md @@ -0,0 +1,404 @@ +## Description + +Retrieves the contents of a database kv_table + +## Positional Parameters +`account` _TEXT_ - The account who owns the kv_table where the smart contract was deployed + +`table` _TEXT_ - The name of the kv_table as specified by the contract abi + +`index_name` _TEXT_ - The name of the kv_table index as specified by the contract abi + +## Options +`-l,--limit` _UINT_ - The maximum number of rows to return + +`-i,--index` _TEXT_ - index value used for point query; encoded as `--encode-type` + +`-L,--lower` _TEXT_ - JSON representation of lower bound index value of `--key` for ranged query (defaults to first). Query result includes rows specified with `--lower` + +`-U,--upper` _TEXT_ - JSON representation of upper bound index value of `--key` for ranged query (defaults to last). Query result does NOT include rows specified with `--upper` + +`--encode-type` _TEXT_ - The encoding type of `--index`, `--lower`, `--upper`; `bytes` for hexadecimal encoded bytes; `string` for string values; `dec` for decimal encoding of (`uint[64|32|16|8]`, `int[64|32|16|8]`, `float64`); `hex` for hexadecimal encoding of (`uint[64|32|16|8]`, `int[64|32|16|8]`, `sha256`, `ripemd160` + +`-b,--binary` _UINT_ - Return the value as BINARY rather than using abi to interpret as JSON + +`-r,--reverse` - Iterate in reverse order; results are returned in reverse order + +`--show-payer` - Show RAM payer + +## Remarks + + * When `--reverse` option is not set, `--upper` is optional; if `--upper` is not set, the result includes the end of the matching rows. + * When `--reverse` option is set, `--lower` is optional; if `--lower` is not set, the result includes the start of the matching rows. + * When the result returns `"more": true`, the remaining rows can be retrieved by setting `--encode_bytes` to `bytes` and `--index`, `--lower` or `--upper` (as applicable, depending on the `--reverse` option) to the value returned in `"next_key": "XYZ"`, where `XYZ` is the next index value in hex. + * When `--index` is used as non-unique secondary index, the result can return multiple rows. + +## Examples + +Point query to return the row that matches eosio name key `boba` from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `primarykey`: +```sh +cleos get kv_table --encode-type name -i boba contr_acct kvtable primarykey -b +``` +```json +{ + "rows": [ + "000000000000600e3d010000000000000004626f6261000000" + ], + "more": false, + "next_key": "" +} +``` + +Point query to return the row that matches decimal key `1` from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `foo` (note that multiple rows could have resulted since `foo` is a non-unique secondary index): +```sh +cleos get kv_table --encode-type dec -i 1 contr_acct kvtable foo +``` +```json +{ + "rows": [ + "000000000000600e3d010000000000000004626f6261" + ], + "more": false, + "next_key": "" +} +``` + +Range query to return all rows starting from eosio name key `bobd` up to `bobh` (exclusive) from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `primarykey`: +```sh +cleos get kv_table --encode-type name -L bobd -U bobh contr_acct kvtable primarykey -b +``` +```json +{ + "rows": [ + "000000000000900e3d040000000000000004626f6264000000", + "000000000000a00e3d050000000000000004626f6265000000", + "000000000000b00e3d060000000000000004626f6266000000", + "000000000000c00e3d070000000000000004626f6267000000" + ], + "more": false, + "next_key": "" +} +``` + +Range query to return all rows (in reverse order) starting from eosio name key `bobh` down to `bobd` (exclusive) from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `primarykey`: +```sh +cleos get kv_table --encode-type name -L bobd -U bobh contr_acct kvtable primarykey -b -r +``` +```json +{ + "rows": [ + "000000000000d00e3d080000000000000004626f6268000000", + "000000000000c00e3d070000000000000004626f6267000000", + "000000000000b00e3d060000000000000004626f6266000000", + "000000000000a00e3d050000000000000004626f6265000000" + ], + "more": false, + "next_key": "" +} +``` + +Range query to return all rows starting from eosio name key `bobg` up to the last row key from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `primarykey`: +```sh +cleos get kv_table --encode-type name -L bobg contr_acct kvtable primarykey -b +``` +```json +{ + "rows": [ + "000000000000c00e3d070000000000000004626f6267000000", + "000000000000d00e3d080000000000000004626f6268000000", + "000000000000e00e3d090000000000000004626f6269000000", + "000000000000f00e3d0a0000000000000004626f626a000000" + ], + "more": false, + "next_key": "" +} +``` + +Range query to return all rows (in reverse order) starting from the last row key down to eosio name key `bobg` (exclusive) from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `primarykey`: +```sh +cleos get kv_table --encode-type name -L bobg contr_acct kvtable primarykey -b -r +``` +```json +{ + "rows": [ + "000000000000f00e3d0a0000000000000004626f626a000000", + "000000000000e00e3d090000000000000004626f6269000000", + "000000000000d00e3d080000000000000004626f6268000000" + ], + "more": false, + "next_key": "" +} +``` + +Range query to return all rows starting from the first row key up to eosio name key `bobe` (exclusive) from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `primarykey`: +```sh +cleos get kv_table --encode-type name -U bobe contr_acct kvtable primarykey -b +``` +```json +{ + "rows": [ + "000000000000600e3d010000000000000004626f6261000000", + "000000000000700e3d020000000000000004626f6262000000", + "000000000000800e3d030000000000000004626f6263000000", + "000000000000900e3d040000000000000004626f6264000000" + ], + "more": false, + "next_key": "" +} +``` + +Range query to return all rows (in reverse order) starting from eosio name key `bobe` down to the first row key from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `primarykey`: +```sh +cleos get kv_table --encode-type name -U bobe contr_acct kvtable primarykey -b -r +``` +```json +{ + "rows": [ + "000000000000a00e3d050000000000000004626f6265000000", + "000000000000900e3d040000000000000004626f6264000000", + "000000000000800e3d030000000000000004626f6263000000", + "000000000000700e3d020000000000000004626f6262000000", + "000000000000600e3d010000000000000004626f6261000000" + ], + "more": false, + "next_key": "" +} +``` + +Range query to return all rows (in reverse order, limit results to 2 rows) starting from eosio name key `bobe` down to the first row key from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `primarykey`: +```sh +cleos get kv_table --encode-type name -U bobe contr_acct kvtable primarykey -b -r -l 2 +``` +```json +{ + "rows": [ + "000000000000a00e3d050000000000000004626f6265000000", + "000000000000900e3d040000000000000004626f6264000000" + ], + "more": true, + "next_key": "3D0E800000000000" +} +``` + +Continue previous range query to return all rows (in reverse order, limit results to 2 rows) starting from hex key `3D0E800000000000` (returned in `next_key` field from previous result) down to the first row key from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `primarykey`: +```sh +cleos get kv_table --encode-type bytes -U 3D0E800000000000 contr_acct kvtable primarykey -b -r -l 2 +``` +```json +{ + "rows": [ + "000000000000800e3d030000000000000004626f6263000000", + "000000000000700e3d020000000000000004626f6262000000" + ], + "more": true, + "next_key": "3D0E600000000000" +} +``` + +Continue previous range query to return all rows (in reverse order, limit results to 2 rows) starting from hex key `3D0E600000000000` (returned in `next_key` field from previous result) down to the first row key from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `primarykey`: +```sh +cleos get kv_table --encode-type bytes -U 3D0E600000000000 contr_acct kvtable primarykey -b -r -l 2 +``` +```json +{ + "rows": [ + "000000000000600e3d010000000000000004626f6261000000" + ], + "more": false, + "next_key": "" +} +``` + +Range query to return all rows starting from decimal key `0` up to `3` (exclusive) from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `foo` (note that key `0` does not exist, so it starts from key `1`): +```sh +cleos get kv_table --encode-type dec -L 0 -U 3 contr_acct kvtable foo -b +``` +```json +{ + "rows": [ + "000000000000600e3d010000000000000004626f6261", + "000000000000700e3d020000000000000004626f6262" + ], + "more": false, + "next_key": "" +} +``` + +Range query to return all rows starting from decimal key `6` up to the last row key from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `foo`: +```sh +cleos get kv_table --encode-type dec -L 6 contr_acct kvtable foo -b +``` +```json +{ + "rows": [ + "000000000000b00e3d060000000000000004626f6266", + "000000000000c00e3d070000000000000004626f6267", + "000000000000d00e3d080000000000000004626f6268", + "000000000000e00e3d090000000000000004626f6269", + "000000000000f00e3d0a0000000000000004626f626a" + ], + "more": false, + "next_key": "" +} +``` + +Range query to return all rows (limit results to 2 rows) starting from decimal key `6` up to the last row key from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `foo`: +```sh +cleos get kv_table --encode-type dec -L 6 contr_acct kvtable foo -b -l 2 +``` +```json +{ + "rows": [ + "000000000000b00e3d060000000000000004626f6266", + "000000000000c00e3d070000000000000004626f6267" + ], + "more": true, + "next_key": "0000000000000008" +} +``` + +Continue previous range query to return all rows (limit results to 2 rows) starting from hex key `0000000000000008` (returned in `next_key` field from previous result, which is also hex for decimal key `8`) up to the last row key from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `foo`: +```sh +cleos get kv_table --encode-type bytes -L 0000000000000008 contr_acct kvtable foo -b -l 2 +``` +```json +{ + "rows": [ + "000000000000d00e3d080000000000000004626f6268", + "000000000000e00e3d090000000000000004626f6269" + ], + "more": true, + "next_key": "000000000000000A" +} +``` + +Continue previous range query to return all rows (limit results to 2 rows) starting from hex key `000000000000000A` (returned in `next_key` field from previous result, which is also hex for decimal key `10`) up to the last row key from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `foo`: +```sh +cleos get kv_table --encode-type bytes -L 000000000000000A contr_acct kvtable foo -b -l 2 +``` +```json +{ + "rows": [ + "000000000000f00e3d0a0000000000000004626f626a" + ], + "more": false, + "next_key": "" +} +``` + +Range query to return all rows (in reverse order) starting from hex key `4` down to `2` (exclusive) from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `foo` (note that hex keys are not correctly specified, or decimal type should be used instead): +```sh +cleos get kv_table --encode-type bytes -L 2 -U 4 contr_acct kvtable foo -b -r +``` +```console +Error 3060003: Contract Table Query Exception +Most likely, the given table doesn't exist in the blockchain. +Error Details: +Invalid index type/encode_type/Index_value: uint64/bytes/{v} +``` + +Range query to return all rows (in reverse order) starting from decimal key `4` down to `2` (exclusive) from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `foo`: +```sh +cleos get kv_table --encode-type dec -L 2 -U 4 contr_acct kvtable foo -b -r +``` +```json +{ + "rows": [ + "000000000000900e3d040000000000000004626f6264", + "000000000000800e3d030000000000000004626f6263" + ], + "more": false, + "next_key": "" +} +``` + +Range query to return all rows (in reverse order) starting from hex key `0000000000000004` down to `0000000000000002` (exclusive) from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `foo` +```sh +cleos get kv_table --encode-type bytes -L 0000000000000002 -U 0000000000000004 contr_acct kvtable foo -b -r +``` +```json +{ + "rows": [ + "000000000000900e3d040000000000000004626f6264", + "000000000000800e3d030000000000000004626f6263" + ], + "more": false, + "next_key": "" +} +``` + +Range query to return all rows starting from string key `boba` up to `bobe` (exclusive) from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `foo` (note that `--lower` and `--upper` values have correct `string` type, but the incorrect index `foo` was used): +```sh +cleos get kv_table --encode-type string -L boba -U bobe contr_acct kvtable foo -b +``` +```json +{ + "rows": [], + "more": false, + "next_key": "" +} +``` + +Range query to return all rows starting from string key `boba` up to `bobe` (exclusive) from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `bar`: +```sh +cleos get kv_table --encode-type string -L boba -U bobe contr_acct kvtable bar -b +``` +```json +{ + "rows": [ + "0186f263c540000000addd235fd05780003d0e60000000", + "0186f263c540000000addd235fd05780003d0e70000000", + "0186f263c540000000addd235fd05780003d0e80000000", + "0186f263c540000000addd235fd05780003d0e90000000" + ], + "more": false, + "next_key": "" +} +``` + +Range query to return all rows (in reverse order) starting from string key `bobe` down to `boba` (exclusive) from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `bar`: +```sh +cleos get kv_table --encode-type string -L boba -U bobe contr_acct kvtable bar -b -r +``` +```json +{ + "rows": [ + "0186f263c540000000addd235fd05780003d0ea0000000", + "0186f263c540000000addd235fd05780003d0e90000000", + "0186f263c540000000addd235fd05780003d0e80000000", + "0186f263c540000000addd235fd05780003d0e70000000" + ], + "more": false, + "next_key": "" +} +``` + +Range query to return all rows (in reverse order, limit results to 2 rows) starting from string key `bobe` down to `boba` (exclusive) from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `bar`: +```sh +cleos get kv_table --encode-type string -L boba -U bobe contr_acct kvtable bar -b -r -l 2 +``` +```json +{ + "rows": [ + "0186f263c540000000addd235fd05780003d0ea0000000", + "0186f263c540000000addd235fd05780003d0e90000000" + ], + "more": true, + "next_key": "626F62630000" +} +``` + +Continue previous range query to return all rows (in reverse order, limit results to 2 rows) starting from hex key `626F62630000` (returned in `next_key` field from previous result, which is also hex for string `bobc`) down to hex key `626F62610000` (hex for string `boba`) (exclusive) from kv_table named `kvtable` owned by `contr_acct` account using kv_table index `bar`: +```sh +cleos get kv_table --encode-type bytes -L 626F62610000 -U 626F62630000 contr_acct kvtable bar -b -r -l 2 +``` +```json +{ + "rows": [ + "0186f263c540000000addd235fd05780003d0e80000000", + "0186f263c540000000addd235fd05780003d0e70000000" + ], + "more": false, + "next_key": "" +} +``` diff --git a/docs/02_cleos/03_command-reference/index.md b/docs/02_cleos/03_command-reference/index.md index ee3eb8ac691..d083920f2aa 100644 --- a/docs/02_cleos/03_command-reference/index.md +++ b/docs/02_cleos/03_command-reference/index.md @@ -4,6 +4,7 @@ Documentation for all `cleos` main commands - [version](version) - Retrieve version information - [create](create) - Create various items, on and off the blockchain - [convert](convert) - Pack and unpack transactions +- [validate](validate) - Validate transactions - [get](get) - Retrieve various items and information from the blockchain - [set](set) - Set or update blockchain state - [transfer](transfer.md) - Transfer tokens from account to account diff --git a/docs/02_cleos/03_command-reference/net/connect.md b/docs/02_cleos/03_command-reference/net/connect.md index 682b5923f4d..be0b2af0b3b 100755 --- a/docs/02_cleos/03_command-reference/net/connect.md +++ b/docs/02_cleos/03_command-reference/net/connect.md @@ -1,17 +1,56 @@ +## Command +```sh +cleos net connect [OPTIONS] host +``` + +**Where:** +* [OPTIONS] = See **Options** in the [**Command Usage**](command-usage) section below. +* host = The hostname:port to connect to + +**Note:** The arguments and options enclosed in square brackets are optional. + ## Description -Start a new connection to a peer +Start a new connection to a specified peer. A node operator can use this command to instruct a node to connect to another peer without restarting the node. + +## Command Usage +The following information shows the different positionals and options you can use with the `cleos net connect` command: + +### Positionals +* `host` _TEXT_ REQUIRED - The hostname:port to connect to + +### Options +* `-h,--help` - Print this help message and exit + +## Requirements +Make sure you meet the following requirements: -**Command** +* Install the currently supported version of `cleos`. +[[info | Note]] +| `cleos` is bundled with the EOSIO software. [Installing EOSIO](../../../00_install/index.md) will also install the `cleos` and `keosd` command line tools. +* You have access to a producing node instance with the [`net_api_plugin`](../../../01_nodeos/03_plugins/net_api_plugin/index.md) loaded. +## Examples +The following examples demonstrate how to use the `cleos net connect` command: + +* Instruct default local node (listening at default http address `http://127.0.0.1:8888`) to connect to peer node listening at p2p address `localhost:9002`: ```sh -cleos net connect +cleos net connect localhost:9002 +``` +**Output:** +```console +"added connection" ``` -**Output** - +* Instruct local node listening at http address `http://127.0.0.1:8001` to connect to peer node listening at p2p address `localhost:9002`: +```sh +cleos -u http://127.0.0.1:8001 net connect localhost:9002 +``` +**Output:** ```console -Usage: cleos net connect host +"added connection" +``` -Positionals: - host TEXT The hostname:port to connect to. +**Note:** If any of the above commands are re-executed, `cleos` returns the following message as expected: +```console +"already connected" ``` diff --git a/docs/02_cleos/03_command-reference/net/disconnect.md b/docs/02_cleos/03_command-reference/net/disconnect.md index 15eafd9ba6a..29c30399610 100755 --- a/docs/02_cleos/03_command-reference/net/disconnect.md +++ b/docs/02_cleos/03_command-reference/net/disconnect.md @@ -1,17 +1,56 @@ +## Command +```sh +cleos net disconnect [OPTIONS] host +``` + +**Where:** +* [OPTIONS] = See **Options** in the [**Command Usage**](command-usage) section below. +* host = The hostname:port to disconnect from + +**Note:** The arguments and options enclosed in square brackets are optional. + ## Description -close an existing connection +Close an existing connection to a specified peer. A node operator can use this command to instruct a node to disconnect from another peer without restarting the node. + +## Command Usage +The following information shows the different positionals and options you can use with the `cleos net disconnect` command: + +### Positionals +* `host` _TEXT_ REQUIRED - The hostname:port to disconnect from + +### Options +* `-h,--help` - Print this help message and exit + +## Requirements +Make sure you meet the following requirements: -**Command** +* Install the currently supported version of `cleos`. +[[info | Note]] +| `cleos` is bundled with the EOSIO software. [Installing EOSIO](../../../00_install/index.md) will also install the `cleos` and `keosd` command line tools. +* You have access to a producing node instance with the [`net_api_plugin`](../../../01_nodeos/03_plugins/net_api_plugin/index.md) loaded. +## Examples +The following examples demonstrate how to use the `cleos net disconnect` command: + +* Instruct default local node (listening at default http address `http://127.0.0.1:8888`) to disconnect from peer node listening at p2p address `localhost:9002`: ```sh -cleos net disconnect +cleos net disconnect localhost:9002 +``` +**Output:** +```console +"connection removed" ``` -**Output** - +* Instruct local node listening at http address `http://127.0.0.1:8001` to disconnect from peer node listening at p2p address `localhost:9002`: +```sh +cleos -u http://127.0.0.1:8001 net disconnect localhost:9002 +``` +**Output:** ```console -Usage: cleos net disconnect host +"connection removed" +``` -Positionals: - host TEXT The hostname:port to disconnect from. +**Note:** If any of the above commands are re-executed, `cleos` returns the following message as expected: +```console +"no known connection for host" ``` diff --git a/docs/02_cleos/03_command-reference/net/peers.md b/docs/02_cleos/03_command-reference/net/peers.md index 4389d5b9514..2814731c758 100755 --- a/docs/02_cleos/03_command-reference/net/peers.md +++ b/docs/02_cleos/03_command-reference/net/peers.md @@ -1,16 +1,112 @@ +## Command +```sh +cleos net peers [OPTIONS] +``` + +**Where:** +* [OPTIONS] = See **Options** in the [**Command Usage**](command-usage) section below. + +**Note:** The arguments and options enclosed in square brackets are optional. + ## Description -status of all existing peers +Returns a list with the status of all peer connections. This command allows a node operator to check the status of a node's peer connections. + +## Command Usage +The following information shows the different positionals and options you can use with the `cleos net peers` command: -**Command** +### Positionals +* `host` _TEXT_ REQUIRED - The hostname:port to disconnect from + +### Options +* `-h,--help` - Print this help message and exit + +## Requirements +Make sure you meet the following requirements: + +* Install the currently supported version of `cleos`. +[[info | Note]] +| `cleos` is bundled with the EOSIO software. [Installing EOSIO](../../../00_install/index.md) will also install the `cleos` and `keosd` command line tools. +* You have access to a producing node instance with the [`net_api_plugin`](../../../01_nodeos/03_plugins/net_api_plugin/index.md) loaded. + +## Examples +The following examples demonstrate how to use the `cleos net peers` command: + +* List the status of all peer connections for a local node listening at http address `http://127.0.0.1:8001`: ```sh -cleos net status +cleos -u http://127.0.0.1:8001 net peers ``` -**Output** +**Output:** +```json +[{ + "peer": "", + "connecting": false, + "syncing": false, + "last_handshake": { + "network_version": 1210, + "chain_id": "60fb0eb4742886af8a0e147f4af6fd363e8e8d8f18bdf73a10ee0134fec1c551", + "node_id": "58ed23173cd4ed89ae90c2e65f5c29bb51e233e78d730d72220b6d84543bfc08", + "key": "EOS1111111111111111111111111111111114T1Anm", + "time": "1621013128445696000", + "token": "0000000000000000000000000000000000000000000000000000000000000000", + "sig": "SIG_K1_111111111111111111111111111111111111111111111111111111111111111116uk5ne", + "p2p_address": "127.0.0.1:9005 - 58ed231", + "last_irreversible_block_num": 127, + "last_irreversible_block_id": "0000007f97323fae098048ae41f22a99238afc5db56cad17f50304919d21e1c2", + "head_num": 128, + "head_id": "0000008072eb0fc043770e44a5db5dedfbd86db9aa5c41b28618f1b9343c2d22", + "os": "osx", + "agent": "EOS Test Agent", + "generation": 4 + } + },{ + "peer": "", + "connecting": false, + "syncing": false, + "last_handshake": { + "network_version": 1210, + "chain_id": "60fb0eb4742886af8a0e147f4af6fd363e8e8d8f18bdf73a10ee0134fec1c551", + "node_id": "2101bd8d770e8eabb3e69cb981db4350b494a04cd5b7a7cea0cd3070aa722306", + "key": "EOS1111111111111111111111111111111114T1Anm", + "time": "1621013129884873000", + "token": "0000000000000000000000000000000000000000000000000000000000000000", + "sig": "SIG_K1_111111111111111111111111111111111111111111111111111111111111111116uk5ne", + "p2p_address": "127.0.0.1:9017 - 2101bd8", + "last_irreversible_block_num": 129, + "last_irreversible_block_id": "0000008117074454dbaa7e8662c6f3d6918e776cc063c45f52b37bdc945ddc5d", + "head_num": 130, + "head_id": "0000008248cc3498b4bbf14a183711d9a73a36517a8069367a343bd4060fed14", + "os": "osx", + "agent": "EOS Test Agent", + "generation": 3 + } + },{ -```console -Usage: cleos net status host + ... -Positionals: - host TEXT The hostname:port to query status of connection + },{ + "peer": "", + "connecting": false, + "syncing": false, + "last_handshake": { + "network_version": 1210, + "chain_id": "60fb0eb4742886af8a0e147f4af6fd363e8e8d8f18bdf73a10ee0134fec1c551", + "node_id": "d9eb85085d09581521d0ec53b87a9657d0176c74fdf8657c56b09a91b3821c6f", + "key": "EOS1111111111111111111111111111111114T1Anm", + "time": "1621013127894327000", + "token": "0000000000000000000000000000000000000000000000000000000000000000", + "sig": "SIG_K1_111111111111111111111111111111111111111111111111111111111111111116uk5ne", + "p2p_address": "127.0.0.1:9002 - d9eb850", + "last_irreversible_block_num": 125, + "last_irreversible_block_id": "0000007d9a3b9cf6a61776ba1bbce226754aefcad664338d2acb5be34cc53a5b", + "head_num": 126, + "head_id": "0000007eccafd4f32a437b5fd8b111b326e2da8d0bcb832036984841b81ab64e", + "os": "osx", + "agent": "EOS Test Agent", + "generation": 4 + } + } +] ``` + +**Note:** The `last_handshake` field contains the chain state of each connected peer as of the last handshake message with the node. For more information read the [Handshake Message](https://developers.eos.io/welcome/latest/protocol/network_peer_protocol#421-handshake-message) in the *Network Peer Protocol* document. diff --git a/docs/02_cleos/03_command-reference/net/status.md b/docs/02_cleos/03_command-reference/net/status.md index 318e21ce35b..f8f45265ecb 100755 --- a/docs/02_cleos/03_command-reference/net/status.md +++ b/docs/02_cleos/03_command-reference/net/status.md @@ -1,31 +1,66 @@ +## Command +```sh +cleos net status [OPTIONS] host +``` + +**Where:** +* [OPTIONS] = See **Options** in the [**Command Usage**](command-usage) section below. +* host = The hostname:port to query status of connection + +**Note:** The arguments and options enclosed in square brackets are optional. + ## Description -status of existing connection +Returns the status of a connected peer. This command allows a node operator to check the status of a node's connected peer. -**Command** +## Command Usage +The following information shows the different positionals and options you can use with the `cleos net status` command: -```sh -cleos net status -``` -**Output** +### Positionals +* `host` _TEXT_ REQUIRED - The hostname:port to query status of connection -```console -Usage: cleos net status host +### Options +* `-h,--help` - Print this help message and exit -Positionals: - host TEXT The hostname:port to query status of connection -``` +## Requirements +Make sure you meet the following requirements: + +* Install the currently supported version of `cleos`. +[[info | Note]] +| `cleos` is bundled with the EOSIO software. [Installing EOSIO](../../../00_install/index.md) will also install the `cleos` and `keosd` command line tools. +* You have access to a producing node instance with the [`net_api_plugin`](../../../01_nodeos/03_plugins/net_api_plugin/index.md) loaded. -Given, a valid, existing `hostname:port` parameter the above command returns a json response looking similar to the one below: +## Examples +The following examples demonstrate how to use the `cleos net status` command: +* List the status of a connected peer listening at p2p address `localhost:9001` for a local node listening at http address `http://127.0.0.1:8002`: + +```sh +cleos -u http://127.0.0.1:8002 net status localhost:9001 ``` +**Output:** +```json { - "peer": "hostname:port", - "connecting": false/true, - "syncing": false/true, + "peer": "localhost:9001", + "connecting": false, + "syncing": false, "last_handshake": { - ... + "network_version": 1210, + "chain_id": "60fb0eb4742886af8a0e147f4af6fd363e8e8d8f18bdf73a10ee0134fec1c551", + "node_id": "7432b032b50a5a3b04a220c48d75f12e5a089405dfee578c3e5b4cf46865e86e", + "key": "EOS1111111111111111111111111111111114T1Anm", + "time": "1620935866018960000", + "token": "0000000000000000000000000000000000000000000000000000000000000000", + "sig": "SIG_K1_111111111111111111111111111111111111111111111111111111111111111116uk5ne", + "p2p_address": "127.0.0.1:9001 - 7432b03", + "last_irreversible_block_num": 184, + "last_irreversible_block_id": "000000b899bd9462ac4697b5d265e47ef5d88d5a66a24a1c2d37de7974fe32f5", + "head_num": 185, + "head_id": "000000b9f79e2394a48738fb3c8c87dac944094648c23818427e1d44375b6034", + "os": "osx", + "agent": "EOS Test Agent", + "generation": 1 } } ``` -The `last_handshake` structure is explained in detail in the [Network Peer Protocol](https://developers.eos.io/welcome/latest/protocol/network_peer_protocol#421-handshake-message) documentation section. \ No newline at end of file +**Note:** The `last_handshake` field contains the chain state of the specified peer as of the last handshake message with the node. For more information read the [Handshake Message](https://developers.eos.io/welcome/latest/protocol/network_peer_protocol#421-handshake-message) in the *Network Peer Protocol* document. diff --git a/docs/02_cleos/03_command-reference/push/push-action.md b/docs/02_cleos/03_command-reference/push/push-action.md index 82e8f657274..fecadad191a 100755 --- a/docs/02_cleos/03_command-reference/push/push-action.md +++ b/docs/02_cleos/03_command-reference/push/push-action.md @@ -8,8 +8,6 @@ Push a transaction with a single action `data` _Type: Text_ - The arguments to the contract -**Output** - ## Options ` -h,--help` - Print this help message and exit @@ -31,3 +29,72 @@ Push a transaction with a single action `--max-net-usage` _UINT_ - set an upper limit on the net usage budget, in bytes, for the transaction (defaults to 0 which means no limit) `--delay-sec` _UINT_ - set the delay_sec seconds, defaults to 0s + +## Output + +For actions that return a value `cleos` has a line which is prefixed with `=>`. + +Let's consider for exemplification the three actions defined below. The first two are `rstring` and `ruint`, which return the value passed as input parameter to them, and the third one is `rtmp` returning a default `tmp` object. + +```c++ +struct tmp { + uint32_t id = 1; + std::vector list{ 1, 2 }; +}; + +[[eosio::action]] +std::string rstring(const std::string & str) { return str; } + +[[eosio::action]] +uint16_t ruint(uint16_t i) { return i; } + +[[eosio::action]] +tmp rtmp() { return tmp{}; } +``` + +The `cleos` commands and their respective outputs are. + +```shell +> cleos push action eosio rstring '{"str":null}' -p eosio@active + +executed transaction: 1e72a3ef1fa01a9c90b870fe86bf31413c6a2f40a2722ca72d9dd707f58851af 96 bytes 116 us +# eosio <= eosio::rstring {"str":""} +=> return value: "" + +> cleos push action eosio rstring '{"str":"test"}' -p eosio@active + +executed transaction: 2484ba021683bb3c6daaabe291ba291e2689d1be899a293d0a09cc68cf0967fb 96 bytes 116 us +# eosio <= eosio::rstring {"str":"test"} +=> return value: "test" + +> cleos push action eosio ruint '{"i":42}' -p eosio@active + +executed transaction: 12c51fd27fad9bb0fa959037a72093ad0b168f5846e916a0e9a295c8416bdf9f 96 bytes 138 us +# eosio <= eosio::ruint {"i":42} +=> return value: 42 +``` + +The `rtmp` action output, returning a default `tmp` object, looks like this: + +```shell +> cleos push action eosio rtmp '{}' -p eosio@active + +executed transaction: 42386687d94695d77d67cf901cf8d1a4f83ac9f89382f0b01f3fb51267cc21cd 96 bytes 110 us +# eosio <= eosio::rtmp "" +=> return value: {"id":1,"list":[1,2]} +``` + +When it is not possible to decode the return value, for example when ABI doesn’t contain information about an action return value, output will look like this: + +```shell +=> return value (hex): 1400000001000770656e64696e6700000000 +``` + +If the length of the return value string is more than 100 chars the string will be truncated to the first 100 chars and three dots `...` will be added at the end. It's similar to what cleos does to action parameters output. + +```shell +> cleos push action eosio rstring '{"str":"qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"}' -p eosio@active +executed transaction: 9fb35c72dbccadb4ff7b72a3a046543fbe2ae36eea70265f2b8356e8192c3f0e 248 bytes 176 us +# eosio <= eosio::rstring {"str":"qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq... +=> return value: "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq... +``` diff --git a/docs/02_cleos/03_command-reference/set/set-account-permission.md b/docs/02_cleos/03_command-reference/set/set-account-permission.md new file mode 100755 index 00000000000..2a2c8bf5593 --- /dev/null +++ b/docs/02_cleos/03_command-reference/set/set-account-permission.md @@ -0,0 +1,108 @@ +## Command +cleos set account permission [OPTIONS] account permission [authority] [parent] + +**Where** +* [OPTIONS] = See Options in Command Usage section below. +* account = The name of the account you want to set. +* permission = The name of the permission you want to set. +* authority = May be a public key, JSON string or filename defining the authority. +* parent = The parent of this permission, defaults to `active`. + +**Note**: The arguments and options enclosed in square brackets are optional. + +## Description +Set or update blockchain account state. Can be used to set parameters dealing with account permissions. + +## Command Usage +The following information shows the different positionals and options you can use with the `cleos set account` command: + +### Positionals: +- `account` _TEXT_ REQUIRED The account to set/delete a permission authority for +- `permission` _TEXT_ REQUIRED The permission name to set/delete an authority for +- `authority` _TEXT_ [delete] NULL, [create/update] public key, JSON string or filename defining the authority, [code] contract name +- `parent` _TEXT_ [create] The permission name of this parents permission, defaults to 'active' + +### Options: +- `-h,--help` Print this help message and exit +- `--add-code` [code] Add 'eosio.code' permission to specified permission authority +- `--remove-code` [code] Remove 'eosio.code' permission from specified permission authority +- `-x`,`--expiration` Set the time in seconds before a transaction expires, defaults to 30s +- `-f`,`--force-unique` Force the transaction to be unique. this will consume extra bandwidth and remove any protections against accidently issuing the same transaction multiple times +- `-s`,`--skip-sign` Specify if unlocked wallet keys should be used to sign transaction +- `-j`,`--json` Print result as json +- `--json-file` _TEXT_ Save result in json format into a file +- `-d`,`--dont-broadcast` Don't broadcast transaction to the network (just print to stdout) +- `--return-packed` Used in conjunction with --dont-broadcast to get the packed transaction +- `-r`,`--ref-block` _TEXT_ Set the reference block num or block id used for TAPOS (Transaction as Proof-of-Stake) +- `--use-old-rpc` Use old RPC push_transaction, rather than new RPC send_transaction +- `-p`,`--permission` _TEXT_ An account and permission level to authorize, as in 'account@permission' (defaults to 'account@active') +- `--max-cpu-usage-ms` _UINT_ Set an upper limit on the milliseconds of cpu usage budget, for the execution of the transaction (defaults to 0 which means no limit) +- `--max-net-usage` _UINT_ Set an upper limit on the net usage budget, in bytes, for the transaction (defaults to 0 which means no limit) +- `--delay-sec` _UINT_ Set the delay_sec seconds, defaults to 0s + +## Requirements +* Install the currently supported version of `cleos`. +[[info | Note]] +| `Cleos` is bundled with the EOSIO software. [Installing EOSIO](../../../00_install/index.md) will also install the `cleos` and `keosd` comand line tools. +* You have access to an EOSIO blockchain. +* You have an EOSIO account and access to the account's private key. + +## Examples + +1. Update the `active` permission key: + +```shell +cleos set account permission alice active EOS5zG7PsdtzQ9achTdRtXwHieL7yyigBFiJDRAQonqBsfKyL3XhC -p alice@owner +``` + +**Where** +* `alice` = The name of the account to update the key. +* `active`= The name of the permission to update the key. +* `EOS5zG7PsdtzQ9achTdRtXwHieL7yyigBFiJDRAQonqBsfKyL3XhC` = The new public key for the authority. +* `-p alice@owner` = The permission used to authorize the transaction. + +**Example Output** +```shell +executed transaction: ab5752ecb017f166d56e7f4203ea02631e58f06f2e0b67103b71874f608793e3 160 bytes 231 us +# eosio <= eosio::updateauth {"account":"alice","permission":"active","parent":"owner","auth":{"threshold":1,"keys":[{"key":"E... +``` + +2. Add the `eosio.code` permission to the contract account `active` permission to enable calling inline actions by the contract account's `active` permission: + +```shell +cleos set account permission alice active --add-code -p alice@active +``` + +**Where** +* `alice` = The name of the account to add `eosio.code`. +* `active`= The name of the permission to add `eosio.code`. +* `--add-code` = Tells the command to add `eosio.code`. +* `-p alice@active` = The permission used to authorize the transaction. + +**Example Output** +```shell +executed transaction: ab5752ecb017f166d56e7f4203ea02631e58f06f2e0b67103b71874f608793e3 160 bytes 231 us +# eosio <= eosio::updateauth {"account":"alice","permission":"active","parent":"owner","auth":{"threshold":1,"keys":[{"key":"E... +``` + +3. Add a custom permission to the **alice** account: + +```shell +cleos set account permission alice customp EOS58wmANoBtT7RdPgMRCGDb37tcCQswfwVpj6NzC55D247tTMU9D active -p alice@active +``` + +**Where** +* `alice` = The name of the account you are adding a custom permission to. +* `customp`= The name of the custom permission. +* `EOS58wmANoBtT7RdPgMRCGDb37tcCQswfwVpj6NzC55D247tTMU9D` = The public key of the custom permission. +* `active` = The parent of the custom permission. +* `-p alice@active` = The permission used to authorize the transaction. + +**Example Output** +```shell +executed transaction: 69c5297571ce3503edb9a1fd8a2f2a5cc1805ad19197a8751ca09093487c3cf8 160 bytes 134 us +# eosio <= eosio::updateauth {"account":"alice","permission":"customp","parent":"active","auth":{"threshold":1,"keys":[{"key":"EOS...``` + +## See Also +- [Accounts and Permissions](https://developers.eos.io/welcome/v2.1/protocol/accounts_and_permissions) protocol document. +- [Creating and Linking Custom Permissions](https://developers.eos.io/welcome/v2.1/smart-contract-guides/linking-custom-permission) tutorial. diff --git a/docs/02_cleos/03_command-reference/set/set-account.md b/docs/02_cleos/03_command-reference/set/set-account.md deleted file mode 100755 index e4ed9f6e32d..00000000000 --- a/docs/02_cleos/03_command-reference/set/set-account.md +++ /dev/null @@ -1,91 +0,0 @@ -## Description -set parameters dealing with account permissions - -[[info | JSON input]] -| This command involves specifying JSON input which depends on underlying class definitions. Therefore, such JSON input is subject to change in future versions of the EOSIO software. - -## Positionals -- `account` _TEXT_ - The account to set/delete a permission authority for -- `permission` _TEXT_ - The permission name to set/delete an authority for -- `authority` _TEXT_ - [delete] NULL, [create/update] public key, JSON string, or filename defining the authority -- `parent` _TEXT_ - [create] The permission name of this parents permission (Defaults to: "active") -## Options - -`-h,--help` Print this help message and exit - -`--add-code` [code] add 'eosio.code' permission to specified permission authority - -`--remove-code` [code] remove 'eosio.code' permission from specified permission authority - -`-x,--expiration` _TEXT_ - set the time in seconds before a transaction expires, defaults to 30s - -`-f,--force-unique` - force the transaction to be unique. this will consume extra bandwidth and remove any protections against accidently issuing the same transaction multiple times - -`-s,--skip-sign` Specify if unlocked wallet keys should be used to sign transaction - -`-d,--dont-broadcast` - Don't broadcast transaction to the network (just print to stdout) - -`-r,--ref-block` _TEXT_ set the reference block num or block id used for TAPOS (Transaction as Proof-of-Stake) - -`-p,--permission` _TEXT_ - An account and permission level to authorize, as in 'account@permission' (defaults to 'account@active') - -`--max-cpu-usage-ms` _UINT_ - set an upper limit on the milliseconds of cpu usage budget, for the execution of the transaction (defaults to 0 which means no limit) - -`--max-net-usage` _UINT_ - set an upper limit on the net usage budget, in bytes, for the transaction (defaults to 0 which means no limit) - -`--delay-sec` _UINT_ - set the delay_sec seconds, defaults to 0s - -## Command -To modify the permissions of an account, you must have the authority over the account and the permission of which you are modifying. - -The first example associates a new key to the active permissions of an account. - -```sh -cleos set account permission test active '{"threshold":1,"keys":[{"key":"EOS8X7Mp7apQWtL6T2sfSZzBcQNUqZB7tARFEm9gA9Tn9nbMdsvBB","weight":1}],"accounts":[{"permission":{"actor":"acc2","permission":"active"},"weight":50}]}' owner -``` -This second example modifies the same account permission, but removes the key set in the last example, and grants active authority of the test account to another account. - -```sh -cleos set account permission test active '{"threshold":1,"keys":[],"accounts":[{"permission":{"actor":"acc1","permission":"active"},"weight":50},{"permission":{"actor":"sandwich","permission":"active"},"weight":1}]}' owner -``` - -The third example demonstrates how to set up permissions for multisig. - -```sh -cleos set account permission test active '{"threshold" : 100, "keys" : [{"key":"EOS8X7Mp7apQWtL6T2sfSZzBcQNUqZB7tARFEm9gA9Tn9nbMdsvBB","weight":25}], "accounts" : [{"permission":{"actor":"sandwich","permission":"active"},"weight":75}]}' owner -``` -The JSON object used in this command is actually composed of two different types of objects - -The authority JSON object ... - -```json -{ - "threshold" : 100, /*An integer that defines cumulative signature weight required for authorization*/ - "keys" : [], /*An array made up of individual permissions defined with an EOS PUBLIC KEY*/ - "accounts" : [] /*An array made up of individual permissions defined with an EOS ACCOUNT*/ -} -``` -...which includes one or more permissions objects. - -```json -/*Set Permission with Key*/ -{ - "permission" : { - "key" : "EOS8X7Mp7apQWtL6T2sfSZzBcQNUqZB7tARFEm9gA9Tn9nbMdsvBB", - "permission" : "active" - }, - weight : 25 /*Set the weight of a signature from this permission*/ -} - -/*Set Permission with Account*/ -{ - "permission" : { - "account" : "sandwich", - "permission" : "active" - }, - weight : 75 /*Set the weight of a signature from this permission*/ -} -``` - -## See Also -- [Accounts and Permissions](https://developers.eos.io/welcome/latest/protocol/accounts_and_permissions) protocol document. diff --git a/docs/02_cleos/03_command-reference/set/set-action-permission.md b/docs/02_cleos/03_command-reference/set/set-action-permission.md new file mode 100755 index 00000000000..8a4701c7bc1 --- /dev/null +++ b/docs/02_cleos/03_command-reference/set/set-action-permission.md @@ -0,0 +1,109 @@ +## Command +cleos set action permission [OPTIONS] account code type requirement + +**Where** +* [OPTIONS] = See Options in Command Usage section below. +* account = The account to set/delete a permission authority for. +* code = The account that owns the code for the action. +* type = The name of the action, note permissions must be set per action. +* requirement = To remove a permission use NULL, To set or update the permission use the name of the permission required for executing the given action. + +**Note**: The arguments and options enclosed in square brackets are optional. + +## Description +Set parameters dealing with account permissions + +## Command Usage +The following information shows the different positionals and options you can use with the `cleos set action permission` command: + +### Positionals: +- `account` _TEXT_ REQUIRED The account to set/delete a permission authority +- `code` _TEXT_ REQUIRED The account that owns the code for the action +- `type` _TEXT_ REQUIRED The type of the action +- `requirement` _TEXT_ REQUIRED [delete] NULL, [set/update] The permission name require for executing the given action + +### Options: +- `-h,--help` Print this help message and exit +- `-x`,`--expiration` Set the time in seconds before a transaction expires, defaults to 30s +- `-f`,`--force-unique` Force the transaction to be unique. this will consume extra bandwidth and remove any protections against accidently issuing the same transaction multiple times +- `-s`,`--skip-sign` Specify if unlocked wallet keys should be used to sign transaction +- `-j`,`--json` Print result as json +- `--json-file` _TEXT_ Save result in json format into a file +- `-d`,`--dont-broadcast` Don't broadcast transaction to the network (just print to stdout) +- `--return-packed` Used in conjunction with --dont-broadcast to get the packed transaction +- `-r`,`--ref-block` _TEXT_ Set the reference block num or block id used for TAPOS (Transaction as Proof-of-Stake) +- `--use-old-rpc` Use old RPC push_transaction, rather than new RPC send_transaction +- `-p`,`--permission` _TEXT_ An account and permission level to authorize, as in 'account@permission' (defaults to 'account@active') +- `--max-cpu-usage-ms` _UINT_ Set an upper limit on the milliseconds of cpu usage budget, for the execution of the transaction (defaults to 0 which means no limit) +- `--max-net-usage` _UINT_ Set an upper limit on the net usage budget, in bytes, for the transaction (defaults to 0 which means no limit) +- `--delay-sec` _UINT_ Set the delay_sec seconds, defaults to 0s + +## Requirements +* Install the currently supported version of `cleos`. +[[info | Note]] +| `Cleos` is bundled with the EOSIO software. [Installing EOSIO](../../../00_install/index.md) will also install the `cleos` and `keosd` comand line tools. +* You have access to an EOSIO blockchain. +* You have an EOSIO account and access to the account's private key. + +## Examples + +1. Set permission on a _hi_ action deployed to the _scontract1_ account so that the _bob_ account's `active` permission and _customp1_ permission are authorized: + +```shell +cleos set action permission bob scontract1 hi customp1 -p bob@active +``` + +**Where** +* `bob` = The name of the account to link the custom permission authority. +* `scontract1`= The name of the account which owns the smart contract. +* `hi` = The name of the action to link to a permission. +* `customp1` = The permission used to authorize the transaction. +* `-p bob@active` = The permission used to authorize setting the permissions. + +**Example Output** +```shell +executed transaction: 4eb4cf3aea232d46e0e949bc273c3f0575be5bdba7b61851ab51d927cf74a838 128 bytes 141 us +# eosio <= eosio::linkauth {"account":"bob","code":"scontract1","type":"hi","requirement":"customp1"} +``` + +2. Set permissions on a _bye_ action deployed to the _scontract1_ account so that the _bob_ account's `active` permission and _customp2_ permission are authorized: + +```shell +cleos set action permission bob scontract1 bye customp2 -p bob@active +``` + +**Where** +* `bob` = The name of the account to link the custom permission authority. +* `scontract1`= The name of the account which owns the smart contract. +* `bye` = The name of the action to link to a permission. +* `customp2` = The permission used to authorize the transaction. +* `-p bob@active` = The permission used to authorize setting the permissions. + +**Example Output** +```shell +executed transaction: 4eb4cf3aea232d46e0e949bc273c3f0575be5bdba7b61851ab51d927cf74a838 128 bytes 141 us +# eosio <= eosio::linkauth {"account":"bob","code":"scontract1","type":"bye","requirement":"customp2"} +``` + +3. To remove the customp1 permission from the _hi_ action: + +```shell +cleos set action permission bob scontract1 hi NULL -p bob@active +``` +**Where** +* `bob` = The name of the account to link the custom permission authority. +* `scontract1`= The name of the account which owns the smart contract. +* `hi` = The name of the action to link to a permission. +* `NULL` = Remove the permission. +* `-p bob@active` = The permission used to authorize the transaction. + +**Example Output** +```shell +executed transaction: 50fe754760a1b8bd0e56f57570290a3f5daa509c090deb54c81a721ee7048201 120 bytes 242 us +# eosio <= eosio::unlinkauth {"account":"bob","code":"scontract1","type":"hi"} +``` + +## See Also +- [Accounts and Permissions](https://developers.eos.io/welcome/v2.1/protocol/accounts_and_permissions) protocol document. +- [Creating and Linking Custom Permissions](https://developers.eos.io/welcome/v2.1/smart-contract-guides/linking-custom-permission) tutorial. + diff --git a/docs/02_cleos/03_command-reference/set/set-action.md b/docs/02_cleos/03_command-reference/set/set-action.md deleted file mode 100755 index cb8199739bf..00000000000 --- a/docs/02_cleos/03_command-reference/set/set-action.md +++ /dev/null @@ -1,73 +0,0 @@ -## Description -Sets or updates an action's state on the blockchain. - -**Command** - -```sh -cleos set action -``` -**Output** - -```console -Usage: cleos set action [OPTIONS] SUBCOMMAND - -Options: - -h,--help Print this help message and exit - -Subcommands: - permission set parmaters dealing with account permissions -``` -**Command** - -```sh -cleos set action permission -``` - -## Positionals - -`account` TEXT The account to set/delete a permission authority for (required - -`code` _Text_ The account that owns the code for the action - -`type` _Type_ the type of the action - -`requirement` _Type_ The permission name require for executing the given action - -## Options -`-h,--help` Print this help message and exit - -`-x,--expiration` _Type:Text_ - set the time in seconds before a transaction expires, defaults to 30s - -`-f,--force-unique` - force the transaction to be unique. this will consume extra bandwidth and remove any protections against accidently issuing the same transaction multiple times - -`-s,--skip-sign` Specify if unlocked wallet keys -should be used to sign transaction - -`-j,--json` print result as json - -`-d,--dont-broadcast` - Don't broadcast transaction to the network (just print to stdout) - -`--return-packed` used in conjunction with --dont-broadcast to get the packed transaction - -`-r,--ref-block` _TEXT_ set the reference block num or block id used for TAPOS (Transaction as Proof-of-Stake) - -`-p,--permission` _Type:Text_ - An account and permission level to authorize, as in 'account@permission' (defaults to 'account@active') - -`--max-cpu-usage-ms` _UINT_ - Set an upper limit on the milliseconds of cpu usage budget, for the execution of the transaction (defaults to 0 which means no limit) - -`--max-net-usage` _UINT_ - Set an upper limit on the net usage budget, in bytes, for the transaction (defaults to 0 which means no limit) - -`--delay-sec` _UINT_ - set the delay_sec seconds, defaults to 0s - -## Usage - -```sh -#Link a `voteproducer` action to the 'voting' permissions -cleos set action permission sandwichfarm eosio.system voteproducer voting -p sandwichfarm@voting - -#Now can execute the transaction with the previously set permissions. -cleos system voteproducer approve sandwichfarm someproducer -p sandwichfarm@voting -``` - -## See Also -- [Accounts and Permissions](https://developers.eos.io/welcome/latest/protocol/accounts_and_permissions) protocol document. diff --git a/docs/02_cleos/03_command-reference/system/system-buyram.md b/docs/02_cleos/03_command-reference/system/system-buyram.md index 028df3f8384..942f7543d4f 100755 --- a/docs/02_cleos/03_command-reference/system/system-buyram.md +++ b/docs/02_cleos/03_command-reference/system/system-buyram.md @@ -1,22 +1,46 @@ + +## Command +cleos system buyram [OPTIONS] payer receiver amount + +**Where** +* [OPTIONS] = See Options in Command Usage section below. +* payer = The account paying for RAM. +* receiver = The account receiving bought RAM. +* amount = The amount of EOS to pay for RAM + +**Note**: The arguments and options enclosed in square brackets are optional. + ## Description -Buy RAM +Use this command to buy RAM for a blockchain account on EOSIO. -## Positional Arguments -- `payer` _TEXT_ - Theaccount paying for RAM +## Command Usage +The following information shows the different positionals and options you can use with the `cleos system buyram` command: + +### Positionals: +- `payer` _TEXT_ - The account paying for RAM - `receiver` _TEXT_ - The account receiving bought RAM - `amount` _TEXT_ - The amount of EOS to pay for RAM -## Options -- `-h,--help` Print this help message and exit -- `-k,--kbytes` buyram in number of kibibytes (KiB) -- `-b,--bytes` buyram in number of bytes -- `-x,--expiration` _TEXT_ - set the time in seconds before a transaction expires, defaults to 30s -- `-f,--force-unique` - force the transaction to be unique. this will consume extra bandwidth and remove any protections against accidently issuing the same transaction multiple times -- `-s,--skip-sign` Specify if unlocked wallet keys should be used to sign transaction -- `-j,--json` - print result as json +### Options +- `-h,--help` - Print this help message and exit +- `-k,--kbytes` - Buyram in number of kibibytes (KiB) +- `-b,--bytes` - Buyram in number of bytes +- `-x,--expiration` _TEXT_ - Set the time in seconds before a transaction expires, defaults to 30s +- `-f,--force-unique` - Force the transaction to be unique. This will consume extra bandwidth and remove any protections against accidently issuing the same transaction multiple times +- `-s,--skip-sign` - Specify if unlocked wallet keys should be used to sign transaction +- `-j,--json` - Print result as json +- `--json-file` _TEXT_ - save result in json format into a file - `-d,--dont-broadcast` - Don't broadcast transaction to the network (just print to stdout) -- `-r,--ref-block` _TEXT_ set the reference block num or block id used for TAPOS (Transaction as Proof-of-Stake) -- `-p,--permission` _TEXT_ - An account and permission level to authorize, as in 'account@permission' (defaults to 'account@active') -- `--max-cpu-usage-ms` _UINT_ - set an upper limit on the milliseconds of cpu usage budget, for the execution of the transaction (defaults to 0 which means no limit) -- `--max-net-usage` _UINT_ - set an upper limit on the net usage budget, in bytes, for the transaction (defaults to 0 which means no limit) -- `--delay-sec` _UINT_ - set the delay_sec seconds, defaults to 0s +- `--return-packed` - Used in conjunction with --dont-broadcast to get the packed transaction +- `-r,--ref-block` _TEXT_ - Set the reference block num or block id used for TAPOS (Transaction as Proof-of-Stake) +- ` --use-old-rpc` - Use old RPC push_transaction, rather than new RPC send_transaction +- `-p,--permission` _TEXT_ - An account and permission level to authorize, as in 'account@permission' (defaults to 'account@active') +- `--max-cpu-usage-ms` _UINT_ - Set an upper limit on the milliseconds of cpu usage budget, for the execution of the transaction (defaults to 0 which means no limit) +- `--max-net-usage` _UINT_ - Set an upper limit on the net usage budget, in bytes, for the transaction (defaults to 0 which means no limit) +- `--delay-sec` _UINT_ - Set the delay_sec seconds, defaults to 0s + +## Requirements +For the prerequisites to run this command see the Before you Begin section of [How to Buy Ram](../02_how-to-guides/how-to-buy-ram.md) + +## Examples +* [How to Buy Ram](../02_how-to-guides/how-to-buy-ram.md) \ No newline at end of file diff --git a/docs/02_cleos/03_command-reference/validate/index.md b/docs/02_cleos/03_command-reference/validate/index.md new file mode 100644 index 00000000000..128256db04f --- /dev/null +++ b/docs/02_cleos/03_command-reference/validate/index.md @@ -0,0 +1,14 @@ +## Description +Validate transactions + +## Commands + +```console +Usage: cleos validate [OPTIONS] SUBCOMMAND + +Options: + -h,--help Print this help message and exit +``` + +## Subcommands +- [signatures](validate-signatures) - Validate signatures and recover public keys diff --git a/docs/02_cleos/03_command-reference/validate/validate-signatures.md b/docs/02_cleos/03_command-reference/validate/validate-signatures.md new file mode 100644 index 00000000000..25235138b95 --- /dev/null +++ b/docs/02_cleos/03_command-reference/validate/validate-signatures.md @@ -0,0 +1,66 @@ +## Description +Validate signatures and recover public keys + +[[info | JSON input]] +| This command involves specifying JSON input which depends on underlying class definitions. Therefore, such JSON input is subject to change in future versions of the EOSIO software. + +## Usage +```sh +cleos validate signatures [OPTIONS] transaction +``` + +## Positional Arguments +- `transaction` _TEXT_ - The JSON string or filename defining the signed transaction to validate + +## Options +- `-c,--chain-id` _TEXT_ - The chain id that will be used in signature verification + +## Example + +```sh +cleos validate signatures --chain-id cf057bbfb72640471fd910bcb67639c22df9f92470936cddc1ade0e2f2e7dc4f '{ "expiration": "2020-04-23T04:47:23", "ref_block_num": 20, "ref_block_prefix": 3872940040, +"max_net_usage_words": 0, "max_cpu_usage_ms": 0, "delay_sec": 0, "context_free_actions": [], "actions": [ { "account": "eosio", "name": "voteproducer", "authorization": [ { "actor": "initb", "permission": "active" } ], "data": "000000008093dd74000000000000000001000000008093dd74" } ], "transaction_extensions": [], "signatures": [ "SIG_K1_Jy81u5yWSE4vGET1cm9TChKrzhAz4QE2hB2pWnUsHQExGafqhVwXtg7a7mbLZwXcon8bVQJ3J5jtZuecJQADTiz2kwcm7c" ], "context_free_data": [] }' +``` +or +```sh +cleos -u https://api.testnet.eos.io validate signatures '{ "expiration": "2020-04-23T04:47:23", "ref_block_num": 20, "ref_block_prefix": 3872940040, +"max_net_usage_words": 0, "max_cpu_usage_ms": 0, "delay_sec": 0, "context_free_actions": [], "actions": [ { "account": "eosio", "name": "voteproducer", "authorization": [ { "actor": "initb", "permission": "active" } ], "data": "000000008093dd74000000000000000001000000008093dd74" } ], "transaction_extensions": [], "signatures": [ "SIG_K1_Jy81u5yWSE4vGET1cm9TChKrzhAz4QE2hB2pWnUsHQExGafqhVwXtg7a7mbLZwXcon8bVQJ3J5jtZuecJQADTiz2kwcm7c" ], "context_free_data": [] }' +``` + +## Output +```console +[ + "EOS7pCywBCz5zw2bc7teCVcT7MEWUr9s749qnYDNPEsBoH32vGqqN" +] +``` + +## See Also +- [How to submit a transaction](../../02_how-to-guides/how-to-submit-a-transaction.md) diff --git a/docs/02_cleos/03_command-reference/wallet/create.md b/docs/02_cleos/03_command-reference/wallet/create.md index b4594459019..463b12d64e9 100755 --- a/docs/02_cleos/03_command-reference/wallet/create.md +++ b/docs/02_cleos/03_command-reference/wallet/create.md @@ -7,4 +7,85 @@ None ## Options - `-n, --name` _TEXT_ - The name of the new wallet - `-f, --file` _TEXT_ - Name of file to write wallet password output to. (Must be set, unless "--to-console" is passed -- `--to-console` - Print password to console \ No newline at end of file +- `--to-console` - Print password to console + +## Command +cleos wallet create [OPTIONS] + +**Where** +* [OPTIONS] = See Options in Command Usage section below. + +**Note**: The arguments and options enclosed in square brackets are optional. + +## Description +Create a new wallet locally. + +## Command Usage +The following information shows the different positionals and options you can use with the `cleos wallet create` command: + +### Positionals: +- none +### Options +`-h`, `--help` - Print this help message and exit +`-n`, `--name` _TEXT_ - The name of the new wallet, if blank the name is `default` +`-f`, `--file` _TEXT_ - Name of file to write wallet password output to. Must be set, unless "--to-console" is passed +`--to-console` - Print password to console. + +## Requirements +* Install the currently supported version of `cleos` and `keosd`. +[[info | Note]] +| `Cleos` and `keosd` are bundled with the EOSIO software. [Installing EOSIO](../../00_install/index.md) will also install the `cleos` and `keosd` command line tools. + +## Examples +1. Create a new wallet called `default` and output the wallet password to the screen +```shell +cleos wallet create --to-console +``` +**Where** +`--to-console` = Tells the `cleos wallet create` command to print the wallet password to the console. + +**Example Output** +```shell +Creating wallet: default +Save password to use in the future to unlock this wallet. +Without password imported keys will not be retrievable. +"PW5HuN7hkCBdMNQAa8z3NuThp3LSDtjv77odQYs5t2wgghJq4MUxo" +``` + +2. Create a new wallet called `my_wallet` and output the wallet password to a file called `my_wallet_password.txt` +```shell +cleos wallet create --name my_wallet --file my_wallet_passwords.txt +``` +**Where** +`--name` my_wallet = Tells the `cleos wallet create` command to create a wallet called `my_wallet_password.txt` +`--file` my_wallet_passwords.txt = Tells the `cleos wallet create` command to store the password in a file called `my_wallet_password.txt`. + +**Example Output** +```shell +Creating wallet: my_wallet +Save password to use in the future to unlock this wallet. +Without password imported keys will not be retrievable. +saving password to my_wallet_passwords.txt +``` + +```shell +cat my_wallet_passwords.txt +``` + +**Example Output** +```shell +PW5JZaEQXoTKWZRg313aTYS7iNh2jMH4Xs7zWS3vrTZ4p5hCSFGFH(base) +``` + +```shell +cleos wallet list +``` + +**Example Output** +```shell +Wallets: +[ + "default *", + "my_wallet *" +] +``` diff --git a/docs/02_cleos/03_command-reference/wallet/import.md b/docs/02_cleos/03_command-reference/wallet/import.md index 38563a1ed1a..a3352b7ac17 100755 --- a/docs/02_cleos/03_command-reference/wallet/import.md +++ b/docs/02_cleos/03_command-reference/wallet/import.md @@ -1,9 +1,55 @@ +## Command +cleos wallet import [OPTIONS] + +**Where** +* [OPTIONS] = See Options in Command Usage section below. + +**Note**: The arguments and options enclosed in square brackets are optional. + ## Description -Imports private key into wallet +Imports private key into wallet. This command will launch `keosd` if it is not already running. -## Positionals -None +## Command Usage +The following information shows the different positionals and options you can use with the `cleos wallet import` command: -## Options +### Positionals: +- none +### Options +- `-h,--help` - Print this help message and exit - `-n, --name` _TEXT_ - The name of the wallet to import key into. -- `--private-key` _TEXT_ - Private key in WIF format to import. \ No newline at end of file +- `--private-key` _TEXT_ - Private key in WIF format to import. + +## Requirements +* Install the currently supported version of `cleos` and `keosd`. +[[info | Note]] +| `Cleos` and `keosd` are bundled with the EOSIO software. [Installing EOSIO](../../00_install/index.md) will also install the `cleos` and `keosd` command line tools. + +## Examples +1. Import a private key to the default wallet. The wallet must be **open** and **unlocked**. +```shell +cleos wallet import +``` + +The command asks for the private key. Enter it. + +```shell +5KDNWQvY2seBPVUz7MiiaEDGTwACfuXu78bwZu7w2UDM9A3u3Fs +``` + +**Example Output** +```shell +private key: imported private key for: EOS5zG7PsdtzQ9achTdRtXwHieL7yyigBFiJDRAQonqBsfKyL3XhC +``` + +2. Import a private key to a named wallet. The wallet must be `open` and `unlocked.` +```shell +cleos wallet import --name my_wallet --private-key 5KDNWQvY2seBPVUz7MiiaEDGTwACfuXu78bwZu7w2UDM9A3u3Fs +``` +**Where** +`--name` my_wallet = Tells the `cleos wallet import` command to import the key to `my_wallet` +`--private-key` 5KDNWQvY2seBPVUz7MiiaEDGTwACfuXu78bwZu7w2UDM9A3u3Fs = Tells the `cleos wallet import` command the private key to import + +**Example Output** +```shell +imported private key for: EOS5zG7PsdtzQ9achTdRtXwHieL7yyigBFiJDRAQonqBsfKyL3XhC +``` diff --git a/docs/02_cleos/index.md b/docs/02_cleos/index.md index 6e0ea9423bd..1c2392aefce 100644 --- a/docs/02_cleos/index.md +++ b/docs/02_cleos/index.md @@ -43,6 +43,7 @@ Subcommands: version Retrieve version information create Create various items, on and off the blockchain convert Pack and unpack transactions + validate Validate transactions get Retrieve various items and information from the blockchain set Set or update blockchain state transfer Transfer tokens from account to account diff --git a/docs/10_utilities/eosio-blocklog.md b/docs/10_utilities/eosio-blocklog.md index b6c7a3ca609..6abe95fcea5 100644 --- a/docs/10_utilities/eosio-blocklog.md +++ b/docs/10_utilities/eosio-blocklog.md @@ -9,13 +9,18 @@ link_text: eosio-blocklog * Generate `blocks.index` from `blocks.log` in blocks directory. * Trim `blocks.log` and `blocks.index` between a range of blocks. * Perform consistency test between `blocks.log` and `blocks.index`. +* Repair `blocks.log` and reconstruct `blocks.index` if corrupted. +* Prune context-free data within given transaction and block number. * Output the results of the operation to a file or `stdout` (default). ## Options +`eosio-blocklog` supports the following options: + Option (=default) | Description -|- `--blocks-dir arg (="blocks")` | The location of the blocks directory (absolute path or relative to the current directory) +`--state-history-dir arg (="state-history")` | The location of the `state-history` directory (absolute path or relative to the current dir) `-o [ --output-file ] arg` | The file to write the generated output to (absolute or relative path). If not specified then output is to `stdout` `-f [ --first ] arg (=0)` | The first block number to log or the first block to keep if `trim-blocklog` specified `-l [ --last ] arg (=4294967295)` | the last block number to log or the last block to keep if `trim-blocklog` specified @@ -23,7 +28,11 @@ Option (=default) | Description `--as-json-array` | Print out JSON blocks wrapped in JSON array (otherwise the output is free-standing JSON objects) `--make-index` | Create `blocks.index` from `blocks.log`. Must give `blocks-dir` location. Give `output-file` relative to current directory or absolute path (default is `/blocks.index`) `--trim-blocklog` | Trim `blocks.log` and `blocks.index`. Must give `blocks-dir` and `first` and/or `last` options. +`--fix-irreversible-blocks` | When the existing block log is inconsistent with the index, allows fixing the block log and index files automatically - it takes the highest indexed block if valid; otherwise, it repairs the block log and reconstructs the index `--smoke-test` | Quick test that `blocks.log` and `blocks.index` are well formed and agree with each other +`--block-num arg (=0)` | The block number which contains the transactions to be pruned +`-t [ --transaction ] arg` | The transaction id to be pruned +`--prune-transactions` | Prune the context free data and signatures from specified transactions of specified block-num `-h [ --help ]` | Print this help message and exit ## Remarks diff --git a/docs/20_upgrade-guide/79_2.0-upgrade-guide.md b/docs/20_upgrade-guide/79_2.0-upgrade-guide.md deleted file mode 100644 index 1a8d5b57ce1..00000000000 --- a/docs/20_upgrade-guide/79_2.0-upgrade-guide.md +++ /dev/null @@ -1,3 +0,0 @@ ---- -link: /20_upgrade-guide/index.md ---- diff --git a/docs/20_upgrade-guide/index.md b/docs/20_upgrade-guide/index.md deleted file mode 100644 index f627e445c39..00000000000 --- a/docs/20_upgrade-guide/index.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -content_title: EOSIO 2.0 Upgrade Guide ---- - -This section contains important instructions for node operators and other EOSIO stakeholders to transition an EOSIO network successfully through an EOSIO version or protocol upgrade. - -## Consensus Protocol Upgrades -EOSIO v2.0.x introduces two new consensus protocol upgrade features (or just protocol features for short): - -1. `WEBAUTHN_KEY`: Adds support for WebAuthn keys and signatures ([#7421](https://github.com/EOSIO/eos/pull/7421)) -2. `WTMSIG_BLOCK_SIGNATURES`: Adds support for weighted threshold multi-signature (WTMsig) authorities for block production ([#7404](https://github.com/EOSIO/eos/pull/7404), [#8002](https://github.com/EOSIO/eos/pull/8002), [#8021](https://github.com/EOSIO/eos/pull/8021)) - -Both of these features require activation through the privileged `preactivate_feature` intrinsic. This is typically facilitated by executing the `eosio::activate` system contract action (requires system contract [v1.7.0](https://github.com/EOSIO/eosio.contracts/releases/tag/v1.7.0) or later) via the `eosio.msig` contract after getting supermajority block producer approval. However, while these protocol features can be activated through coordination of the block producers alone, **it is still critical to give sufficient time for all nodes to upgrade to v2.0.x before activating any of the above two protocol features**. Activation of any of these two protocol features will immediately fork out any nodes that do not support the protocol features (e.g. any nodes running a version of nodeos earlier than v2.0.x). - -Both of the above protocol features only add new behavior to the EOSIO blockchain. They are not expected to change existing behavior in a significant way that would cause existing contracts to break. Existing applications and block explorers are also unlikely to break upon activation of these protocol features. However, these new features enable new data structures to be utilized on-chain which applications and especially block explorers need to be prepared to handle. For example, activation of the `WEBAUTHN_KEY` protocol feature means that an `eosio::newaccount` or `eosio::updateauth` action could now include WebAuthn public keys as part of the provided `authority`. So, an application dealing with these actions (e.g. a general authenticator for EOSIO transactions) will need to be prepared to support the serialization of WebAuthn public keys appearing in a place where a `public_key` ABI type is expected. Another example is how the activation of the `WTMSIG_BLOCK_SIGNATURES` protocol feature causes the `new_producers` field of a block header to always remain empty, even when a proposed producer schedule gets promoted to pending; upon activation of the `WTMSIG_BLOCK_SIGNATURES` protocol feature, the pending schedule data is instead stored (when present) within the `header_extensions` of the block header. So, block explorers expecting to show these pending producer schedules to users will need to update their code accordingly. As usual, test networks provide a great environment to try existing contracts, applications, services, etc. with the changes of the above protocol features to determine what breaks and what needs to be updated. - -For more details on the `WEBAUTH_KEY` protocol feature, see the section titled [WebAuthn Support](#webauthn-support-7421). For more details on the `WTMSIG_BLOCK_SIGNATURES` protocol feature, see the section titled [Weighted Threshold Multi-signature (WTMsig) Block Signing Authorities](#weighted-threshold-multi-signature-block-signing-authorities-7403). - -### WebAuthn Support ([#7421](https://github.com/EOSIO/eos/pull/7421)) -WebAuthn support within EOSIO is now available for developers to begin testing WebAuthn based transaction signing in their EOSIO applications. Private EOSIO chains can [start](https://github.com/EOSIO/eosio-webauthn-example-app) using WebAuthn based signatures with this release. We also anticipate releasing additional demos which will attempt to outline how Dapp authors can leverage WebAuthn for contract actions meriting higher levels of security as a second factor authorization mechanism. - -### Weighted Threshold Multi-Signature Block Signing Authorities ([#7403](https://github.com/EOSIO/eos/issues/7403)) -Block producers must be able to provide high availability for their core service of running the blockchain (aka producing blocks). A common approach to achieve this is redundant infrastructure that efficiently maintains block production, in the event of a hardware malfunction or networking issues. Weighted Threshold Multi-Signature Block Production is the first of many features that seek to provide block producers with a complete high availability solution. - -Current consensus rules require exactly one cryptographic block signing key per block producer. This key, whether stored on disk and loaded via software or protected with a hardware wallet, represents a single-point-of-failure for the operations of a block producer. If that key is lost or access to the hardware module that contains it is temporarily unavailable, the block producer has no choice but to drop blocks, impacting the whole network’s throughput. - -To improve the security and scalability of block production, weighted threshold multi-signature (WTMsig) block signing authorities provides a permission layer that allows for multiple block signing keys in a flexible scheme that will enable redundant block signing infrastructure to exist without sharing any sensitive data. - -An updated version of the system contract is required to enable block producers to use WTMsig block signing authorities. Version [v1.9.0-rc1](https://github.com/EOSIO/eosio.contracts/releases/tag/v1.9.0-rc1) of eosio.contracts adds a new `regproducer2` action to the eosio.system contract which enables a block producer candidate to register a WTMsig block signing authority. (That version of the system contract can only be deployed on an EOSIO chain that has activated the `WTMSIG_BLOCK_SIGNATURES` protocol feature.) More details are available at: [#7404](https://github.com/EOSIO/eos/pull/7404), [#8002](https://github.com/EOSIO/eos/pull/8002), [#8021](https://github.com/EOSIO/eos/pull/8021). - -## Upgrading from previous versions of EOSIO - -EOSIO v2.0.x has made changes to the state database that prevent it from working with existing state databases in v1.8.x and earlier. So upgrading from earlier versions of EOSIO requires importing the state from a portable snapshot. - -Furthermore, the changes to the state database necessitate a version bump in the portable snapshots generated by EOSIO v2.0.x from v2 to v3. However, unlike the upgrade of portable snapshot versions in EOSIO v1.8.x from v1 to v2, EOSIO v2.0.x is able to import v2 portable snapshots. This means that it is not necessary to replay the blockchain from genesis to upgrade from EOSIO v1.8.x to v2.0.x. (One exception is if the operator of the node is using the deprecated history_plugin and wishes to preserve that history.) - -Finally, EOSIO v2.0.x has also upgraded the version of the irreversible blocks log to v3. However, older versions of the blocks log are still supported, so there is no need to do anything special to handle existing blocks log files. - -### Upgrading from v1.8.x - -If the node uses the deprecated history_plugin (and the operator of the node wishes to preserve this history), the only option to upgrade is to replay the blockchain from genesis. - -Users of the state_history_plugin (SHiP) do not need to replay from genesis because the state history logs are portable and contain versioned data structures within. However, upgrading a node that uses state history without a full replay means that the state history log will contain a mix of versions for any upgrade types. For example, the changes in v2.0.x modify the `global_property_object` in the database state and so the state history log could contain a `global_property_object_v0` type (for the part of the history before the local node upgrade) and a `global_property_object_v1` type (for the part of the history after the local node upgrade). This should not cause problems for any history fillers that have been updated to support both versions of the type. However, operators should be aware that this can lead to the log file contents being slightly different across different nodes even if they all start and stop at the same blocks and have not enabled `trace-history-debug-mode`. (Replaying a v2.0.x node with state_history_plugin enabled from genesis would guarantee that the state history logs do not contain the `global_property_object_v0` type.) - -The following instructions should be followed to upgrade nodeos from v1.8.x to v2.0.x without a full replay (after making appropriate backup copies): -1. Obtain a compatible portable snapshot of the state. A v2 or v3 portable snapshot can be downloaded from a trusted source. Alternatively, one can use an existing v1.8.x node to generate a v2 portable snapshot by launching nodeos with `--read-mode=irreversible --plugin=eosio::producer_api_plugin` command-line options and then using the `/v1/producer/create_snapshot` RPC endpoint to generate a portable snapshot (e.g. run the command `curl -X POST http:/127.0.0.1:8888/v1/producer/create_snapshot -d '{}' | jq`). -2. Shut down nodeos and delete the `blocks/reversible` and `state` sub-directories within the data directory. -3. Launch nodeos v2.0.x from the generated snapshot using the `--snapshot` command line option and give it time to import the snapshot while starting up (this could take several minutes). (Subsequent launches of nodeos should not use the `--snapshot` option.) - -### Upgrading from v2.0.0-rc1, v2.0.0-rc2, or v2.0.0-rc3 - -Node operators should consider upgrading v2.0.0-rc1, v2.0.0-rc2, and v2.0.0-rc3 nodes to v2.0.0 as soon as possible. They can follow normal upgrade procedures for the upgrade. There should be no need to do a replay or import from a snapshot. diff --git a/docs/20_upgrade-guide/81_1.8-upgrade-guide.md b/docs/20_upgrade-guides/1.8-upgrade-guide.md similarity index 94% rename from docs/20_upgrade-guide/81_1.8-upgrade-guide.md rename to docs/20_upgrade-guides/1.8-upgrade-guide.md index 33a24e6ee08..a5936f472ca 100644 --- a/docs/20_upgrade-guide/81_1.8-upgrade-guide.md +++ b/docs/20_upgrade-guides/1.8-upgrade-guide.md @@ -1,5 +1,5 @@ --- -content_title: EOSIO 1.8 Consensus Protocol Upgrade Process +content_title: EOSIO 1.8+ Consensus Protocol Upgrade Process --- This guide is intended to instruct node operators on the steps needed to successfully transition an EOSIO network through a consensus protocol upgrade (also known as a "hard fork") with minimal disruption to users. @@ -121,7 +121,7 @@ Once the `PREACTIVATE_FEATURE` protocol feature has been activated, the [new sys Block explorers, exchanges, and applications building on the blockchain can all follow the four-step processes described above to upgrade their nodes in time and ensure their services continue when the first protocol upgrade is activated. However, they should also be aware that certain protocol features change the behavior of existing operations on the blockchain, and in some cases also slightly change the structure of blocks and transactions. -**First**, v1.8 changes the structure of transaction traces, even prior to the activation of any protocol features. Clients consuming transaction and action traces made available through [`history_plugin`](../03_plugins/history_plugin/index.md), [`mongo_db_plugin`](../03_plugins/mongo_db_plugin/index.md), or [`state_history_plugin`](../03_plugins/state_history_plugin/index.md) should be aware of the changes made to the trace structure (see details at [#7044](https://github.com/EOSIO/eos/pull/7044) and [#7108](https://github.com/EOSIO/eos/pull/7108)). Clients consuming the trace output of the `push_transaction` RPC from the chain API should not need to do anything since the output of that RPC should be backwards compatible. However, they are encouraged to replace usage of `push_transaction` with the new RPC [`send_transaction`](https://developers.eos.io/eosio-nodeos/reference#send_transaction) which uses the new flat structure to store the action traces. +**First**, v1.8 changes the structure of transaction traces, even prior to the activation of any protocol features. Clients consuming transaction and action traces made available through [`history_plugin`](../03_plugins/history_plugin/index.md), `mongo_db_plugin`, or [`state_history_plugin`](../03_plugins/state_history_plugin/index.md) should be aware of the changes made to the trace structure (see details at [#7044](https://github.com/EOSIO/eos/pull/7044) and [#7108](https://github.com/EOSIO/eos/pull/7108)). Clients consuming the trace output of the `push_transaction` RPC from the chain API should not need to do anything since the output of that RPC should be backwards compatible. However, they are encouraged to replace usage of `push_transaction` with the new RPC [`send_transaction`](https://developers.eos.io/eosio-nodeos/reference#send_transaction) which uses the new flat structure to store the action traces. The [`state_history_plugin`](../03_plugins/state_history_plugin/index.md) has also changed its API and the structure of the files it stores on disk in a backwards incompatible way in v1.8. These changes reflect, among other things, the transaction trace structural changes and the data structure changes made within the chain state database to support the new protocol features. Consumers of the [`state_history_plugin`](../03_plugins/state_history_plugin/index.md) will need to be updated to work with the new changes in v1.8. diff --git a/docs/20_upgrade-guides/index.md b/docs/20_upgrade-guides/index.md new file mode 100644 index 00000000000..1013692a810 --- /dev/null +++ b/docs/20_upgrade-guides/index.md @@ -0,0 +1,7 @@ +--- +content_title: EOSIO Upgrade Guides +--- + +This section contains important instructions for node operators and other EOSIO stakeholders to transition an EOSIO network successfully through an EOSIO version or protocol upgrade. + +* [1.8 Upgrade Guide](1.8-upgrade-guide.md) diff --git a/docs/30_release-notes/89_v2.0.10.md b/docs/30_release-notes/89_v2.0.10.md deleted file mode 100644 index 5ddad16cc70..00000000000 --- a/docs/30_release-notes/89_v2.0.10.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -content_title: EOSIO v2.0.10 Release Notes -link_text: v2.0.10 ---- - -This release contains security, stability, and miscellaneous fixes. - -## Security bug fixes - -### Consolidated Security Fixes for v2.0.10 ([#10091](https://github.com/EOSIO/eos/pull/10091)) -- Fix issue with account query db that could result in incorrect data or hung processes -- Implement a Subjective CPU billing system that helps P2P and API nodes better respond to extreme network congestion - -Note: These security fixes are relevant to all nodes on EOSIO blockchain networks. - -### Notes on Subjective CPU Billing - -This system consists of two primary features: a subjective (node local) view of spent CPU resources that are not yet accounted for by the blockchain that allows individual nodes to predict what resource consumption will be and, a subjective penalty system to offset work done in service of erroneous or malicious transactions. - -The subjective view of CPU resources will synchronize with the resources present in the blockchain as it discovers the true CPU billing for transactions it has already accounted for. - -The system will also accumulate CPU resources spent on failing transactions that will not be relayed in a decaying "subjective penalty" which can protect the individual nodes from abusive actors while remaining tolerant to occasional mistakes. - -Subjective billing defaults to active and can be disabled with the `disable-subjective-billing` configuration in `config.ini` or on the command line. - -## Stability bug fixes -- ([#9985](https://github.com/EOSIO/eos/pull/9985)) EPE-389 net_plugin stall during head catchup - merge release/2.0.x - -## Other changes -- ([#9894](https://github.com/EOSIO/eos/pull/9894)) EOS VM OC: Support LLVM 11 - 2.0 -- ([#9911](https://github.com/EOSIO/eos/pull/9911)) add step to the pipeline to build and push to dockerhub on release br… -- ([#9944](https://github.com/EOSIO/eos/pull/9944)) Create eosio-debug-build Pipeline -- ([#9969](https://github.com/EOSIO/eos/pull/9969)) Updating name for the new Docker hub repo EOSIO instead EOS -- ([#9971](https://github.com/EOSIO/eos/pull/9971)) Fix pinned builds error due to obsolete LLVM repo -- ([#10015](https://github.com/EOSIO/eos/pull/10015)) [release 2.0.x] Fix LRT triggers -- ([#10026](https://github.com/EOSIO/eos/pull/10026)) EPE-165: Improve logic for unlinkable blocks while sync'ing -- ([#10047](https://github.com/EOSIO/eos/pull/10047)) Reduce Docker Hub Manifest Queries -- ([#10088](https://github.com/EOSIO/eos/pull/10088)) [release 2.0.x] Specify boost version for unpinned MacOS 10.14 builds - -## Documentation -- ([#10011](https://github.com/EOSIO/eos/pull/10011)) [docs] Update various cleos how-tos and fix index - 2.0 diff --git a/docs/30_release-notes/90_v2.0.9.md b/docs/30_release-notes/90_v2.0.9.md deleted file mode 100644 index f3c27400392..00000000000 --- a/docs/30_release-notes/90_v2.0.9.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -content_title: EOSIO v2.0.9 Release Notes -link_text: v2.0.9 ---- - -This release contains security and miscellaneous fixes. - -## Security bug fixes - -### Consolidated Security Fixes for 2.0.9 ([#9841](https://github.com/EOSIO/eos/pull/9841)) -- Fixes to packed_transaction cache -- Transaction account fail limit refactor - -Note: These security fixes are relevant to all nodes on EOSIO blockchain networks. - -## Other Changes -- ([#9803](https://github.com/EOSIO/eos/pull/9803)) Fix stdout console logging: Merge back #9582 to 2.0.x diff --git a/docs/30_release-notes/91_v2.0.8.md b/docs/30_release-notes/91_v2.0.8.md deleted file mode 100644 index ecf5bad0b07..00000000000 --- a/docs/30_release-notes/91_v2.0.8.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -content_title: EOSIO v2.0.8 Release Notes -link_text: v2.0.8 ---- - -This release contains security, stability, and miscellaneous fixes. - -## Security bug fixes - -### Consolidated Security Fixes for 2.0.8 ([#9745](https://github.com/EOSIO/eos/pull/9745)) - -- Adjust eos-vm-oc string intrinsic to perform as intended. -- Adjust CPU validation logic for unapplied transactions. - -Note: These security fixes are relevant to all nodes on EOSIO blockchain networks. - -## Stability bug fixes -- ([#9370](https://github.com/EOSIO/eos/pull/9370)) set medium priority for process signed block - 2.0.x -- ([#9382](https://github.com/EOSIO/eos/pull/9382)) Handle case newaccount in Account Query DB -- ([#9412](https://github.com/EOSIO/eos/pull/9412)) port the fix of flight bytes bug into release/2.0.x -- ([#9423](https://github.com/EOSIO/eos/pull/9423)) Update to fc with variant blob fix - 2.0 -- ([#9441](https://github.com/EOSIO/eos/pull/9441)) Fix app() shutdown - 2.0 -- ([#9516](https://github.com/EOSIO/eos/pull/9516)) Keep http_plugin_impl alive while connection objects are alive - 2.0 -- ([#9624](https://github.com/EOSIO/eos/pull/9624)) Fixing typos on injected params for `producer_plugin::log_failed_tran… - -## Other Changes -- ([#9304](https://github.com/EOSIO/eos/pull/9304)) relaxing the on_notify constraint to * -- ([#9311](https://github.com/EOSIO/eos/pull/9311)) Track Source Files Excluded from Code Coverage Reports -- ([#9314](https://github.com/EOSIO/eos/pull/9314)) Prevent an older version of g++ to build eosio -- ([#9334](https://github.com/EOSIO/eos/pull/9334)) Add missing comma in loggers array -- ([#9399](https://github.com/EOSIO/eos/pull/9399)) [2.0.x] Fix docker tags when building forked PRs -- ([#9638](https://github.com/EOSIO/eos/pull/9638)) Migrate CI from Docker Hub to Amazon ECR -- ([#9657](https://github.com/EOSIO/eos/pull/9657)) CI: Fix Serial Test Bug + Simplification + UX -- ([#9665](https://github.com/EOSIO/eos/pull/9665)) Add "Testing Changes" Section to Pull Request Template - -## Documentation -- ([#9323](https://github.com/EOSIO/eos/pull/9323)) [docs] Remove unneeded options for nodeos replays - 2.0 -- ([#9322](https://github.com/EOSIO/eos/pull/9322)) [docs] Remove redundant nodeos replay example - 2.0 -- ([#9373](https://github.com/EOSIO/eos/pull/9373)) [docs] Fix broken link in Wallet API plugin - 2.0 -- ([#9464](https://github.com/EOSIO/eos/pull/9464)) [docs] Create nodeos concepts folder and rearrange folders - 2.0 -- ([#9479](https://github.com/EOSIO/eos/pull/9479)) [docs] Add explainers on CFD, eosio utilities, eosio-blocklog - 2.0 -- ([#9487](https://github.com/EOSIO/eos/pull/9487)) [docs] Minor edits on CFD explainer and eosio-blocklog reference - 2.0 -- ([#9488](https://github.com/EOSIO/eos/pull/9488)) [docs] Fix how-tos for delegating cpu/net with cleos - 2.0 -- ([#9491](https://github.com/EOSIO/eos/pull/9491)) [docs] Add EOSIO upgrade guide 2.0 to dev portal -- ([#9492](https://github.com/EOSIO/eos/pull/9492)) Add explicit left nav link for eosio 2.0 upgrade guide - 2.0 -- ([#9496](https://github.com/EOSIO/eos/pull/9496)) [docs] Add eosio 2.0 release notes to dev portal - 2.0 -- ([#9498](https://github.com/EOSIO/eos/pull/9498)) [docs] Add trace_api_util reference to eosio utilities docs - 2.0 -- ([#9503](https://github.com/EOSIO/eos/pull/9503)) [docs] Add slices, trace log, clog format explainers to Trace API plugin - 2.0 -- ([#9584](https://github.com/EOSIO/eos/pull/9584)) [docs] Update cleos get table reference - 2.0 -- ([#9592](https://github.com/EOSIO/eos/pull/9592)) [docs] Various additions/fixes to cleos reference - 2.0 -- ([#9602](https://github.com/EOSIO/eos/pull/9602)) [docs] Fix broken anchor link on MacOS build from source - 2.0 -- ([#9627](https://github.com/EOSIO/eos/pull/9627)) [docs] Update get_table_* reference in Chain API - 2.0 -- ([#9753](https://github.com/EOSIO/eos/pull/9753)) [docs] Update URL in net_api_plugin description - 2.0 -- ([#9754](https://github.com/EOSIO/eos/pull/9754)) [docs] Update some chain_api_plugin descriptions - 2.0 -- ([#9756](https://github.com/EOSIO/eos/pull/9756)) [docs] Remove sudo command from install/uninstall script instructions - 2.0 - -## Thanks! -Special thanks to the community contributors that submitted patches for this release: -- @nsjames diff --git a/docs/30_release-notes/92_v2.0.7.md b/docs/30_release-notes/92_v2.0.7.md deleted file mode 100644 index b92acb66f11..00000000000 --- a/docs/30_release-notes/92_v2.0.7.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -content_title: EOSIO v2.0.7 Release Notes -link_text: v2.0.7 ---- - -This release contains stability and miscellaneous fixes. - -## Stability bug fixes -- ([#9223](https://github.com/EOSIO/eos/pull/9223)) Fix log of pending block producer - 2.0 - -## Changes - -### The following logger has been added: `transaction_failure_tracing`. ([#9252](https://github.com/EOSIO/eos/pull/9252)) - -A detailed log that emits failed verdicts from relay nodes on the P2P network. In addition, it adds a logging statement for a failed signature condition. - -### New config file option: `max-nonprivileged-inline-action-size`. ([#9262](https://github.com/EOSIO/eos/pull/9262)) - -This option is the cap for the size of an inline action above which a transaction will subjectively fail; the default value is 4KB. - -## Other Changes -- ([#9170](https://github.com/EOSIO/eos/pull/9170)) Fix onblock trace tracking - 2.0 -- ([#9201](https://github.com/EOSIO/eos/pull/9201)) [2.0.x] Anka version bump -- ([#9265](https://github.com/EOSIO/eos/pull/9265)) Remove Concurrency Groups for Scheduled Builds - -## Documentation -- ([#9124](https://github.com/EOSIO/eos/pull/9124)) Update the authority example -- ([#9275](https://github.com/EOSIO/eos/pull/9275)) [docs] New threshold for non privileged inline actions - 2.0 -- ([#9288](https://github.com/EOSIO/eos/pull/9288)) [docs] Fix character formatting in nodeos CLI option - 2.0 -- ([#9290](https://github.com/EOSIO/eos/pull/9290)) [docs] Correct Producer API title in RPC reference - 2.0 diff --git a/docs/30_release-notes/93_v2.0.6.md b/docs/30_release-notes/93_v2.0.6.md deleted file mode 100644 index 97815d56bf9..00000000000 --- a/docs/30_release-notes/93_v2.0.6.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -content_title: EOSIO v2.0.6 Release Notes -link_text: v2.0.6 ---- - -This release contains security, stability, and miscellaneous fixes. - -## Security bug fixes - -- ([#9172](https://github.com/EOSIO/eos/pull/9172)) Escape Unicode C1 control code points. - -Note: These security fixes are relevant to API nodes on EOSIO blockchain networks. - -## Stability bug fixes - -- ([#9065](https://github.com/EOSIO/eos/pull/9065)) Fix for cleos and keosd race condition - 2.0 -- ([#9089](https://github.com/EOSIO/eos/pull/9089)) make ship WA key serialization match expected serialization - 2.0 -- ([#9095](https://github.com/EOSIO/eos/pull/9095)) fix gcc10 build due to libyubihsm problem - 2.0 -- ([#9127](https://github.com/EOSIO/eos/pull/9127)) Fix onblock handling in trace_api_plugin - 2.0 -- ([#9129](https://github.com/EOSIO/eos/pull/9129)) GCC 8.3 on CentOS 7 compiler workaround - 2.0 -- ([#9128](https://github.com/EOSIO/eos/pull/9128)) Restore abi_serializer backward compatibility - 2.0 - -## Changes - -### Add more information in trace-api-plugin responses for better usage. ([#9005](https://github.com/EOSIO/eos/pull/9005)) - -Adds `transaction_mroot`, `action_mroot` and `schedule_version` in block trace. Also adds `status`, `cpu_usage_us`, `net_usage_words`, `signatures`, and `transaction_header` in transaction trace. - -### New RPC endpoint **`get_accounts_by_authorizers`** ([#8899](https://github.com/EOSIO/eos/pull/8899)) - -New optional RPC endpoint **`POST /v1/chain/get_accounts_by_authorizers`** added to `chain_api_plugin` that provides a super-set of the deprecated `history_api_plugin`'s `get_key_accounts` and `get_controlled_accounts` RPC methods. - -Flag to enable endpoint (default false): **`--enable-account-queries`** - -## Other Changes - -- ([#8975](https://github.com/EOSIO/eos/pull/8975)) failing nodeos_run_test when core symbol is not SYS - 2.0 -- ([#9002](https://github.com/EOSIO/eos/pull/9002)) Support Triggering a Build that Runs ALL Tests in One Build -- ([#9007](https://github.com/EOSIO/eos/pull/9007)) Improved reporting in nodeos_forked_chain_lr_test - 2.0.x -- ([#9013](https://github.com/EOSIO/eos/pull/9013)) Bugfix for uninitialized variable in cleos - 2.0 -- ([#9009](https://github.com/EOSIO/eos/pull/9009)) Upgrade CLI11 to 1.9.0 - 2.0 -- ([#9028](https://github.com/EOSIO/eos/pull/9028)) Fix keosd auto-launching after CLI11 upgrade - 2.0 -- ([#9035](https://github.com/EOSIO/eos/pull/9035)) For Release 2.0 - Updated the priority of the APIs in producer_api_plugin and net_api_plugin to MEDIUM_HIGH -- ([#9049](https://github.com/EOSIO/eos/pull/9049)) add rapidjson license to install - 2.0 -- ([#9052](https://github.com/EOSIO/eos/pull/9052)) Print stderr if keosd_auto_launch_test.py fails - 2.0 -- ([#9060](https://github.com/EOSIO/eos/pull/9060)) Fix uninitialized struct members used as CLI flags - 2.0 -- ([#9062](https://github.com/EOSIO/eos/pull/9062)) Fix timedelta and strftime usage - 2.0 -- ([#9078](https://github.com/EOSIO/eos/pull/9078)) Update date in LICENSE - 2.0 -- ([#9063](https://github.com/EOSIO/eos/pull/9063)) add help text to wasm-runtime - 2.0.x -- ([#9084](https://github.com/EOSIO/eos/pull/9084)) Add support for specifing a logging.json to keosd - 2.0 -- ([#9082](https://github.com/EOSIO/eos/pull/9082)) Add change type to pull request template - 2.0 -- ([#8899](https://github.com/EOSIO/eos/pull/8899)) Account Query DB : Proposal to maintain get_(key|controlled)_accounts [2.0] -- ([#9103](https://github.com/EOSIO/eos/pull/9103)) Add default contract name clarifier in how to deploy smart contract - 2.0 -- ([#9109](https://github.com/EOSIO/eos/pull/9109)) [2.0.x] Bump Anka plugin version and timeouts. -- ([#9115](https://github.com/EOSIO/eos/pull/9115)) Simplify create_snapshot POST request - 2.0 -- ([#9110](https://github.com/EOSIO/eos/pull/9110)) Update algorithm for determining number of parallel jobs - 2.0 - -## Documentation - -- ([#8980](https://github.com/EOSIO/eos/pull/8980)) Add nodeos RPC API index, improve nodeos implementation doc, fix link - 2.0 -- ([#8995](https://github.com/EOSIO/eos/pull/8995)) Update example logging.json - 2.0 -- ([#9102](https://github.com/EOSIO/eos/pull/9102)) Fix inaccurate nodeos reference in wallet_api_plugin - 2.0 -- ([#9116](https://github.com/EOSIO/eos/pull/9116)) Replace inaccurate wording in how to replay from snapshot - 2.0 -- ([#9113](https://github.com/EOSIO/eos/pull/9113)) Add trace_api logger to docs - 2.0 -- ([#9142](https://github.com/EOSIO/eos/pull/9142)) Add missing reference to RPC API index [docs] - 2.0 -- ([#9141](https://github.com/EOSIO/eos/pull/9141)) Fix Trace API reference request/response inaccuracies [docs] - 2.0 -- ([#9144](https://github.com/EOSIO/eos/pull/9144)) Fix title case issue in keosd how-to [docs] - 2.0 -- ([#9145](https://github.com/EOSIO/eos/pull/9145)) Add conditional step in state history plugin how-to [docs] - 2.0 - -## Thanks! - -Special thanks to the community contributors that submitted patches for this release: -- @cc32d9 -- @oldcold diff --git a/docs/30_release-notes/94_v2.0.5.md b/docs/30_release-notes/94_v2.0.5.md deleted file mode 100644 index 2a912715ad1..00000000000 --- a/docs/30_release-notes/94_v2.0.5.md +++ /dev/null @@ -1,65 +0,0 @@ ---- -content_title: EOSIO v2.0.5 Release Notes -link_text: v2.0.5 ---- - -This release contains security, stability, and miscellaneous fixes. - -## Security bug fixes - -### Consolidated Security Fixes for 2.0.5 ([#8983](https://github.com/EOSIO/eos/pull/8983)) - -- EOS-VM security fixes - -Note: These security fixes are relevant to all nodes on EOSIO blockchain networks. - -## Stability bug fixes - -- ([#8826](https://github.com/EOSIO/eos/pull/8826)) trace_api_plugin yield timeout - 2.0 -- ([#8836](https://github.com/EOSIO/eos/pull/8836)) fix potential leak in OC's wrapped_fd move assignment op - 2.0 - -## Changes - -### Trace API Compressed Data Log Support ([#8826](https://github.com/EOSIO/eos/pull/8826), [#8837](https://github.com/EOSIO/eos/pull/8837), [#8881](https://github.com/EOSIO/eos/pull/8881)) - -Compressed file support was added to `trace_api_plugin`. See ([#8837](https://github.com/EOSIO/eos/pull/8837)) for more details. - -The RPC call `v1/trace_api/get_block` now has "async" http support. Therefore, executing `get_block` no longer runs on the main application thread but on the configurable `http-threads` thread pool. - -Additionally, `trace_api_plugin` now respects `http-max-response-time-ms` for limiting response time of RPC call `v1/trace_api/get_block`. It is very likely that the default value of `http-max-response-time-ms` will not be appropriate for large blocks and will need to be increased. - -## Other Changes - -- ([#8822](https://github.com/EOSIO/eos/pull/8822)) Merge minimize logging changes to 2.0.x -- ([#8823](https://github.com/EOSIO/eos/pull/8823)) yield_function for abi_serializer - 2.0 -- ([#8855](https://github.com/EOSIO/eos/pull/8855)) Improve too many bytes in flight error info - 2.0 -- ([#8861](https://github.com/EOSIO/eos/pull/8861)) HTTP Plugin async APIs [2.0] -- ([#8873](https://github.com/EOSIO/eos/pull/8873)) Fix spurious HTTP related test failure [2.0] (round 3) -- ([#8883](https://github.com/EOSIO/eos/pull/8883)) wabt: don't search for python because we don't run tests - 2.0 -- ([#8884](https://github.com/EOSIO/eos/pull/8884)) Correctly Sanitize git Branch and Tag Names -- ([#8894](https://github.com/EOSIO/eos/pull/8894)) Increase get info priority to medium high -- ([#8889](https://github.com/EOSIO/eos/pull/8889)) Sync from snapshot - 2.0 -- ([#8906](https://github.com/EOSIO/eos/pull/8906)) Remove assert check for error code 400 - release 2.0.x -- ([#8944](https://github.com/EOSIO/eos/pull/8944)) noop change to macos-10.14-unpinned.sh to regen CI image, 2.0 -- ([#8941](https://github.com/EOSIO/eos/pull/8941)) replace boost::bind with std::bind, fixing boost 1.73beta builds - 2.0 -- ([#8954](https://github.com/EOSIO/eos/pull/8954)) llvm 10 support for EOS VM OC - 2.0 -- ([#8949](https://github.com/EOSIO/eos/pull/8949)) Replace bc with shell arithmetic - 2.0 -- ([#8962](https://github.com/EOSIO/eos/pull/8962)) tests/get_table_tests.cpp: incorrect use of CORE_SYM_STR - 2.0 -- ([#8963](https://github.com/EOSIO/eos/pull/8963)) Make /bin/df ignore $BLOCKSIZE - 2.0 -- ([#8952](https://github.com/EOSIO/eos/pull/8952)) Fix SHIP block delay - 2.0 -- ([#8972](https://github.com/EOSIO/eos/pull/8972)) Add possibility to run .cicd scripts from different environments (2.0.x Backport) -- ([#8968](https://github.com/EOSIO/eos/pull/8968)) Support Running ALL Tests in One Build - -## Documentation changes - -- ([#8825](https://github.com/EOSIO/eos/pull/8825)) remove leading $ chars from shell codeblocks in README.md - 2.0 -- ([#8835](https://github.com/EOSIO/eos/pull/8835)) Trace API documentation update - 2.0 -- ([#8843](https://github.com/EOSIO/eos/pull/8843)) Fix double titles for release 2.0.x -- ([#8845](https://github.com/EOSIO/eos/pull/8845)) [docs] trace api reference api correction - 2.0 -- ([#8918](https://github.com/EOSIO/eos/pull/8918)) Updates to manual build instructions - 2.0 - -## Thanks! - -Special thanks to the community contributors that submitted patches for this release: -- @cc32d9 -- @maoueh diff --git a/docs/30_release-notes/95_v2.0.4.md b/docs/30_release-notes/95_v2.0.4.md deleted file mode 100644 index 6906b87069b..00000000000 --- a/docs/30_release-notes/95_v2.0.4.md +++ /dev/null @@ -1,88 +0,0 @@ ---- -content_title: EOSIO v2.0.4 Release Notes -link_text: v2.0.4 ---- - -This release contains stability and miscellaneous fixes. - -## Deprecation Notices - -The `read-only` option for the `read-mode` parameter in `nodeos` has been deprecated. It is possible to achieve the same behavior with `read-mode = head`, `p2p-accept-transactions = false`, and `api-accept-transactions = false`. See the sub-section "Accept transactions options" below for more details. - -Please refer to the [Consolidated EOSIO Deprecations List](https://github.com/EOSIO/eos/issues/7597) for the currently active set of deprecation notices. - -## Stability bug fixes - -- ([#8684](https://github.com/EOSIO/eos/pull/8684)) Net plugin sync priority - 2.0 -- ([#8729](https://github.com/EOSIO/eos/pull/8729)) Get info priority - 2.0 - -## Changes - -### Trace API Plugin ([#8800](https://github.com/EOSIO/eos/pull/8800)) - -This release contains the first official release of the Trace API Plugin. This plugin is an optional addition to `nodeos` that stores a tailored view of the transactions and actions that execute on the chain retrievable at a block level. The Trace API focuses on operational maintainability storing data on the filesystem instead of in RAM like the deprecated `history-plugin` and organizing that data such that operators can easily prune old data without disrupting operations. - -For more information, see the PR notes and the official documentation. - -### Exit transaction early when there is insufficient account CPU ([#8638](https://github.com/EOSIO/eos/pull/8638)) - -`nodeos` no longer considers a transaction for inclusion in a block in the process of being produced if the billed account(s) do not have sufficient CPU available to cover the previously estimated CPU usage of the transaction (only if a previous estimate for CPU usage is available). - -### Produce block immediately if resource limits are exhausted ([#8651](https://github.com/EOSIO/eos/pull/8651), [#8673](https://github.com/EOSIO/eos/pull/8673)) - -`nodeos` now immediately produces a block if either the CPU or NET usage thresholds are exceeded. This change includes a fix for dropping late blocks starting 50ms earlier than the block production window. - -New options: -* `max-block-cpu-usage-threshold-us`: -Threshold (in microseconds) of CPU block production to consider block full; when accumulated CPU usage within a block is less than `max-block-cpu-usage-threshold-us` away from `max-block-cpu-usage`, the block can be produced immediately. Default value is 5000. -* `max-block-net-usage-threshold-bytes`: -Threshold (in bytes) of NET block production to consider block full; when accumulated NET usage within a block is less than `max-block-net-usage-threshold-us` away from `max-block-net-usage`, the block can be produced immediately. Default value is 1024. - -### Accept transactions options ([#8702](https://github.com/EOSIO/eos/pull/8702)) - -New options: -* `p2p-accept-transactions`: Allow transactions received over p2p -network to be evaluated and relayed if valid. Default is true. -* `api-accept-transactions`: Allow API transactions to be evaluated -and relayed if valid. Default is true. - -Provides ability to have a `read-mode = head` with `p2p-accept-transactions = false` and `api-accept-transactions = true`. This combination creates an efficient API node that is not burdened with processing P2P transactions. - -The same behavior of the now deprecated `read-mode = read-only` can be achieved with `read-mode = head` by setting `p2p-accept-transactions = false` and `api-accept-transactions = false`. - -**WARNING:** Use of `read-mode = irreversible` now requires setting `p2p-accept-transactions = false` and `api-accept-transactions = false` to avoid assertion at startup. - -### Relay block early ([#8701](https://github.com/EOSIO/eos/pull/8701)) - -Improve block relaying performance when a block is from a trusted producer or if `nodeos` is running in light validation mode. This is achieved by relaying the block as soon as block header validation is complete (but before full block application/validation). - -## Other Changes - -- ([#8654](https://github.com/EOSIO/eos/pull/8654)) Fix format message. - 2.0 -- ([#8668](https://github.com/EOSIO/eos/pull/8668)) Add troubleshooting item for PREACTIVATE_FEATURE protocol -- ([#8689](https://github.com/EOSIO/eos/pull/8689)) incoming-defer-ratio description - 2.0 -- ([#8695](https://github.com/EOSIO/eos/pull/8695)) [2.0.x] Community PR tweaks. -- ([#8700](https://github.com/EOSIO/eos/pull/8700)) [2.0.x] Base images pipeline. -- ([#8714](https://github.com/EOSIO/eos/pull/8714)) [2.0.x] Actions rerun fixes. -- ([#8710](https://github.com/EOSIO/eos/pull/8710)) Add block producing explainer doc - 2.0 -- ([#8721](https://github.com/EOSIO/eos/pull/8721)) Fix multiple version protocol test intermittent failure - 2.0 -- ([#8727](https://github.com/EOSIO/eos/pull/8727)) Update the getting started link [merge 2] -- ([#8752](https://github.com/EOSIO/eos/pull/8752)) chain_api_plugin swagger file - 2.0 -- ([#8756](https://github.com/EOSIO/eos/pull/8756)) Fixes #8600 clean up nodeos options section -- ([#8757](https://github.com/EOSIO/eos/pull/8757)) link cleos net status reference doc with the peer network protocol doc -- ([#8590](https://github.com/EOSIO/eos/pull/8590)) db_size_api_plugin swagger file - 2.0 -- ([#8591](https://github.com/EOSIO/eos/pull/8591)) net_api_plugin swagger file - 2.0 -- ([#8592](https://github.com/EOSIO/eos/pull/8592)) producer_api_plugin swagger file - 2.0 -- ([#8593](https://github.com/EOSIO/eos/pull/8593)) test_control_api_plugin swagger file - 2.0 -- ([#8754](https://github.com/EOSIO/eos/pull/8754)) swagger configuration for docs - 2.0 -- ([#8762](https://github.com/EOSIO/eos/pull/8762)) Fix broken link in producer plugin docs - 2.0 -- ([#8763](https://github.com/EOSIO/eos/pull/8763)) Fix wasm-runtime option parameters - 2.0 -- ([#8765](https://github.com/EOSIO/eos/pull/8765)) Add Incoming-defer-ratio description - 2.0 -- ([#8767](https://github.com/EOSIO/eos/pull/8767)) Fix other blocks.log callout - 2.0 -- ([#8768](https://github.com/EOSIO/eos/pull/8768)) Improve create account description - 2.0 -- ([#8782](https://github.com/EOSIO/eos/pull/8782)) link to librt when using posix timers - 2.0 -- ([#8781](https://github.com/EOSIO/eos/pull/8781)) free unknown EOS VM OC codegen versions from the code cache - 2.0 -- ([#8794](https://github.com/EOSIO/eos/pull/8794)) disable EOS VM on non-x86 platforms - 2.0 -- ([#8803](https://github.com/EOSIO/eos/pull/8803)) Expire blacklisted scheduled transactions by LIB time - 2.0 -- ([#8811](https://github.com/EOSIO/eos/pull/8811)) Add initial Trace API plugin docs to nodeos - 2.0 -- ([#8814](https://github.com/EOSIO/eos/pull/8814)) Add RPC Trace API plugin reference to nodeos - 2.0 diff --git a/docs/30_release-notes/96_v2.0.3.md b/docs/30_release-notes/96_v2.0.3.md deleted file mode 100644 index 1ba0a6ffb22..00000000000 --- a/docs/30_release-notes/96_v2.0.3.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -content_title: EOSIO v2.0.3 Release Notes -link_text: v2.0.3 ---- - -This release contains security, stability, and miscellaneous fixes. - -## Security bug fixes - -### Consolidated Security Fixes for 2.0.3 ([#8643](https://github.com/EOSIO/eos/pull/8643)) - -- Add deadline to base58 encoding. - -Note: These security fixes are relevant to all nodes on EOSIO blockchain networks. - -## Stability bug fixes - -- ([#8617](https://github.com/EOSIO/eos/pull/8617)) Init net_plugin member variables - 2.0 - -## Other Changes - -- ([#8606](https://github.com/EOSIO/eos/pull/8606)) Skip sync from genesis and resume from state test on tagged builds -- ([#8612](https://github.com/EOSIO/eos/pull/8612)) [2.0.x] Actions for community PRs. -- ([#8633](https://github.com/EOSIO/eos/pull/8633)) remove brew's python@2 install - 2.0 -- ([#8636](https://github.com/EOSIO/eos/pull/8636)) bump script's macos version check to 10.14 - 2.0 - -## Deprecation notice reminder - -Please refer to the [Consolidated EOSIO Deprecations List](https://github.com/EOSIO/eos/issues/7597) for the currently active set of deprecation notices. diff --git a/docs/30_release-notes/97_v2.0.2.md b/docs/30_release-notes/97_v2.0.2.md deleted file mode 100644 index c9707f171cc..00000000000 --- a/docs/30_release-notes/97_v2.0.2.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -content_title: EOSIO v2.0.2 Release Notes -link_text: v2.0.2 ---- - -This release contains security, stability, and miscellaneous fixes. - -## Security bug fixes - -### Consolidated Security Fixes for 2.0.2 ([#8595](https://github.com/EOSIO/eos/pull/8595)) - -- Restrict allowed block signature types. - -Note: These security fixes are relevant to all nodes on EOSIO blockchain networks. - -## Stability bug fixes - -- ([#8526](https://github.com/EOSIO/eos/pull/8526)) Handle socket close before async callback - 2.0 -- ([#8546](https://github.com/EOSIO/eos/pull/8546)) Net plugin dispatch - 2.0 -- ([#8552](https://github.com/EOSIO/eos/pull/8552)) Net plugin unlinkable blocks - 2.0 -- ([#8560](https://github.com/EOSIO/eos/pull/8560)) Backport of 8056 to 2.0 -- ([#8561](https://github.com/EOSIO/eos/pull/8561)) Net plugin post - 2.0 -- ([#8564](https://github.com/EOSIO/eos/pull/8564)) Delayed production time - 2.0 - -## Changes - -### Limit block production window ([#8571](https://github.com/EOSIO/eos/pull/8571), [#8578](https://github.com/EOSIO/eos/pull/8578)) - -The new options `cpu-effort-percent` and `last-block-cpu-effort-percent` now provide a mechanism to restrict the amount of time a producer is processing transactions for inclusion into a block. It also controls the time a producer will finalize/produce and transmit a block. Block construction now always begins at whole or half seconds for the next block. - -### Stricter signature parsing - -Versions of EOSIO prior to v2.0.x accept keys and signatures containing extra data at the end. In EOSIO v2.0.x, the Base58 string parser performs additional sanity checks on keys and signatures to make sure they do not contain more data than necessary. These stricter checks cause nodes to reject signatures generated by some transaction signing libraries; eosjs v20 is known to generate proper signatures. For more information see issue [#8534](https://github.com/EOSIO/eos/issues/8534). - -## Other Changes - -- ([#8555](https://github.com/EOSIO/eos/pull/8555)) Drop late check - 2.0 -- ([#8557](https://github.com/EOSIO/eos/pull/8557)) Read-only with drop-late-block - 2.0 -- ([#8568](https://github.com/EOSIO/eos/pull/8568)) Timestamp watermark slot - 2.0.x -- ([#8577](https://github.com/EOSIO/eos/pull/8577)) [release 2.0.x] Documentation patch 1 update -- ([#8583](https://github.com/EOSIO/eos/pull/8583)) P2p read only - 2.0 -- ([#8589](https://github.com/EOSIO/eos/pull/8589)) Producer plugin log - 2.0 - -## Deprecation notice reminder - -Please refer to the [Consolidated EOSIO Deprecations List](https://github.com/EOSIO/eos/issues/7597) for the currently active set of deprecation notices. diff --git a/docs/30_release-notes/88_v2.0.11.md b/docs/30_release-notes/97_v2.1.0-rc3.md similarity index 65% rename from docs/30_release-notes/88_v2.0.11.md rename to docs/30_release-notes/97_v2.1.0-rc3.md index 0bbd3fa7f99..64fe6c43c04 100644 --- a/docs/30_release-notes/88_v2.0.11.md +++ b/docs/30_release-notes/97_v2.1.0-rc3.md @@ -1,4 +1,4 @@ --- link: /30_release-notes/index.md -link_text: v2.0.11 +link_text: v2.1.0-rc3 --- diff --git a/docs/30_release-notes/98_v2.0.1.md b/docs/30_release-notes/98_v2.0.1.md deleted file mode 100644 index 755ef919c3c..00000000000 --- a/docs/30_release-notes/98_v2.0.1.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -content_title: EOSIO v2.0.1 Release Notes -link_text: v2.0.1 ---- - -This release contains security, stability, and miscellaneous fixes. - -## Security bug fixes - -### Consolidated Security Fixes for 2.0.1 ([#8514](https://github.com/EOSIO/eos/pull/8514)) - -- Earlier block validation for greater security. -- Improved handling of deferred transactions during block production. -- Reduce net plugin logging and handshake size limits. - -Note: These security fixes are relevant to all nodes on EOSIO blockchain networks. - -## Stability bug fixes - -- ([#8471](https://github.com/EOSIO/eos/pull/8471)) Remove new block id notify feature - 2.0 -- ([#8472](https://github.com/EOSIO/eos/pull/8472)) Report block header diff when digests do not match - 2.0 -- ([#8496](https://github.com/EOSIO/eos/pull/8496)) Drop late blocks - 2.0 -- ([#8510](https://github.com/EOSIO/eos/pull/8510)) http_plugin shutdown - 2.0 - -## Other Changes - -- ([#8430](https://github.com/EOSIO/eos/pull/8430)) Update fc to fix crash in logging -- ([#8435](https://github.com/EOSIO/eos/pull/8435)) [release/2.0.x] Update README.md and hotfix documentation links -- ([#8452](https://github.com/EOSIO/eos/pull/8452)) [2.0.x] [CI/CD] Boost will not install without SDKROOT -- ([#8457](https://github.com/EOSIO/eos/pull/8457)) [2.0.x] reverting fc -- ([#8458](https://github.com/EOSIO/eos/pull/8458)) [2.0.x] Pipeline file for testing the build script -- ([#8467](https://github.com/EOSIO/eos/pull/8467)) [2.0.x] Switching to using the EOSIO fork of anka-buildkite-plugin for security reasons -- ([#8515](https://github.com/EOSIO/eos/pull/8515)) [2.0.x] Don't trigger LRT a second time - -## Deprecation notice reminder - -Please refer to the [Consolidated EOSIO Deprecations List](https://github.com/EOSIO/eos/issues/7597) for the currently active set of deprecation notices. diff --git a/docs/30_release-notes/98_v2.1.0-rc2.md b/docs/30_release-notes/98_v2.1.0-rc2.md new file mode 100644 index 00000000000..fcbd145c71d --- /dev/null +++ b/docs/30_release-notes/98_v2.1.0-rc2.md @@ -0,0 +1,31 @@ +--- +content_title: EOSIO v2.1.0-rc2 Release Notes +link_text: v2.1.0-rc2 +--- + +This is a ***RELEASE CANDIDATE*** for version 2.1.0. + +This release contains security, stability, and miscellaneous fixes. + +## Security bug fixes +- ([#9828](https://github.com/EOSIO/eos/pull/9828)) Fix packed transaction version conversion -- Release 2.1.x + +Note: This security fix is relevant to all nodes on EOSIO blockchain networks. + +## Stability bug fixes +- ([#9811](https://github.com/EOSIO/eos/pull/9811)) Fix the truncate bug in Ship - 2.1 +- ([#9812](https://github.com/EOSIO/eos/pull/9812)) Fix snapshot test_compatible_versions failure and reenable it - release/2.1.x +- ([#9813](https://github.com/EOSIO/eos/pull/9813)) fix balance transfer issue - release/2.1.x +- ([#9829](https://github.com/EOSIO/eos/pull/9829)) Fix ship truncate problem with stride +- ([#9835](https://github.com/EOSIO/eos/pull/9835)) Fix Ship backward compatibility issue +- ([#9838](https://github.com/EOSIO/eos/pull/9838)) fix populating some information for get account + +## Other changes +- ([#9801](https://github.com/EOSIO/eos/pull/9801)) Fix build script problem with older version of cmake +- ([#9802](https://github.com/EOSIO/eos/pull/9802)) Add CentOS 8 Package Builder Step +- ([#9820](https://github.com/EOSIO/eos/pull/9820)) Reduce logging for failed http plugin calls - 2.1 + +## Documentation +- ([#9818](https://github.com/EOSIO/eos/pull/9818)) [docs] Fix blockvault plugin explainer and C++ reference links - 2.1 +- ([#9806](https://github.com/EOSIO/eos/pull/9806)) [docs] Corrections to nodeos storage and read modes - 2.1 +- ([#9808](https://github.com/EOSIO/eos/pull/9808)) [docs] 2.1.x update link to chain plug-in to be relative diff --git a/docs/30_release-notes/99_v2.0.0.md b/docs/30_release-notes/99_v2.0.0.md deleted file mode 100644 index 6462f86fad7..00000000000 --- a/docs/30_release-notes/99_v2.0.0.md +++ /dev/null @@ -1,377 +0,0 @@ ---- -content_title: EOSIO v2.0.0 Release Notes -link_text: v2.0.0 ---- - -This release contains security, stability, and miscellaneous fixes. - -This release also includes an EOSIO consensus upgrade. Please see the "Consensus Protocol Upgrades" section below for details. These protocol upgrades necessitate a change to the state database structure which requires a nodeos upgrade process that involves more steps than in the case of most minor release upgrades. For details on the upgrade process, see the "Upgrading from previous versions of EOSIO" section below. - -## Security bug fixes - -### Consolidated Security Fixes for 2.0.0 ([#8420](https://github.com/EOSIO/eos/pull/8420)) - -- Limit size of response to API requests -- EOS VM fixes - -Note: These security fixes are relevant to all nodes on EOSIO blockchain networks. - -The above is in addition to the other security fixes that were introduced in prior release candidates for 2.0.0: - -- ([#8195](https://github.com/EOSIO/eos/pull/8195)) Consolidated Security Fixes for 2.0.0-rc2 -- ([#8344](https://github.com/EOSIO/eos/pull/8344)) Consolidated Security Fixes for 2.0.0-rc3 - -## Stability bug fixes - -- ([#8399](https://github.com/EOSIO/eos/pull/8399)) Net plugin sync check - 2.0 - -This above is in addition to the other stability fixes that were introduced in prior release candidates for 2.0.0: - -- ([#8084](https://github.com/EOSIO/eos/pull/8084)) Net plugin remove read delays - 2.0 -- ([#8099](https://github.com/EOSIO/eos/pull/8099)) net_plugin remove sync w/peer check - 2.0 -- ([#8120](https://github.com/EOSIO/eos/pull/8120)) Net plugin sync fix - 2.0 -- ([#8229](https://github.com/EOSIO/eos/pull/8229)) Net plugin sync -- ([#8285](https://github.com/EOSIO/eos/pull/8285)) Net plugin handshake -- ([#8298](https://github.com/EOSIO/eos/pull/8298)) net_plugin lib sync -- ([#8303](https://github.com/EOSIO/eos/pull/8303)) net_plugin boost asio error handling -- ([#8305](https://github.com/EOSIO/eos/pull/8305)) net_plugin thread protection peer logging variables -- ([#8311](https://github.com/EOSIO/eos/pull/8311)) Fix race in fc::message_buffer and move message_buffer_tests to fc. -- ([#8307](https://github.com/EOSIO/eos/pull/8307)) reset the new handler - -## Changes - -### EOS VM: New High Performance WASM Runtimes ([#7974](https://github.com/EOSIO/eos/pull/7974), [#7975](https://github.com/EOSIO/eos/pull/7975)) -Three new WASM runtimes are available in this release: EOS VM Interpreter, EOS VM Just In Time Compiler (JIT), and EOS VM Optimized Compiler. - -EOS VM Interpreter is a low latency interpreter and is included to enable future support for smart contract debuggers. EOS VM JIT is a low latency single pass compiler for x86_64 platforms. To use EOS VM Interpreter or EOS VM JIT, set the `wasm-runtime` to either `eos-vm` or `eos-vm-jit` respectively - -EOS VM Optimized Compiler is a high performance WASM _tier-up_ runtime available on the x86_64 Linux platform that works in conjunction with the configured baseline runtime (such as EOS VM JIT). When enabled via `eos-vm-oc-enable`, actions on a contract are initially dispatched to the baseline runtime while EOS VM Optimized Compiler does an optimized compilation in the background. Up to the configured `eos-vm-oc-compile-threads` compilations can be ongoing simultaneously. Once the background compilation is complete, actions on that contract will then be run with the optimized compiled code from then on. This optimized compile can take a handful of seconds but since it is performed in the background it does not cause delays. Optimized compilations are also saved to a new data file (`code_cache.bin` in the data directory) so when restarting nodeos there is no need to recompile previously compiled contracts. - -None of the EOS VM runtimes require a replay nor activation of any consensus protocol upgrades. They can be switched between (including enabling and disabling EOS VM Optimized Compiler) at will. - -At this time, **block producers should consider all running EOS VM JIT as the WASM runtime on their block producing nodes**. Other non-producing nodes should feel free to use the faster EOS VM Optimized Compiler runtime instead. - -### Consensus Protocol Upgrades - -Refer to this section on the [Upgrade Guide: Consensus Protocol Upgrades](../20_upgrade-guide/index.md#consensus-protocol-upgrades). - -### Multi-threaded net_plugin -We have added multi-threading support to net_plugin. Almost all processing in the net_plugin (block propagation, transaction processing, block/transaction packing/unpacking etc.) are now handled by separate threads distinct from the main application thread. This significantly improves transaction processing and block processing performance on multi-producer EOSIO networks. The `net-threads` arg (defaults to 2) controls the number of worker threads in net_plugin thread pool.([#6845](https://github.com/EOSIO/eos/pull/6845), [#7598](https://github.com/EOSIO/eos/pull/7598), [#7392](https://github.com/EOSIO/eos/pull/7392), [#7786](https://github.com/EOSIO/eos/pull/7786) and related optimizations are available in: [#7686](https://github.com/EOSIO/eos/pull/7686), [#7785](https://github.com/EOSIO/eos/pull/7785), [#7721](https://github.com/EOSIO/eos/pull/7721), [#7825](https://github.com/EOSIO/eos/pull/7825), and [#7756](https://github.com/EOSIO/eos/pull/7756)). - -### Chain API Enhancements ([#7530](https://github.com/EOSIO/eos/pull/7530)) -The `uint128` and `int128` ABI types are now represented as decimal numbers rather than the old little-endian hexadecimal representation. This means that the JSON representation of table rows returned by the `get_table_rows` RPC will represent fields using this type differently than in prior versions. It also means that the `lower_bound` and `upper_bound` fields for `get_table_rows` RPC requests that search using a `uint128` secondary index will need to use the new decimal representation. This change makes the ABI serialization for `uint128` and `int128` ABI types consistent with [eosjs](https://github.com/EOSIO/eosjs) and [abieos](http://github.com/EOSIO/abieos). - -The `get_table_rows` RPC when used with secondary index types like `sha256`, `i256`, and `ripemd160` had bugs that scrambled the bytes in the `lower_bound` and `upper_bound` input field. This release now fixes these issues allowing clients to properly search for tables rows using these index types. - -A new field `next_key` has been added to the response of the `get_table_rows` RPC which represents the key of the next row (in the same format as `lower_bound` and `upper_bound`) that was not able to be returned in the response due to timeout limitations or the user-specified limit. The value of the `next_key` can be used as the `lower_bound` input for subsequent requests in order to retrieve a range of a large number of rows that could not be retrieved within just a single request. - -## Upgrading from previous versions of EOSIO - -Refer to this section on the [Upgrade Guide: Upgrading from previous versions of EOSIO](../20_upgrade-guide/index.md#upgrading-from-previous-versions-of-eosio). - -## Deprecation and Removal Notices - -### WAVM removed ([#8407](https://github.com/EOSIO/eos/pull/8407)) - -The WAVM WebAssembly runtime was deprecated several months ago with the release of [EOSIO v2.0.0-rc1](https://github.com/EOSIO/eos/releases/tag/v2.0.0-rc1) which introduced EOS VM. Now with the release of a stable version 2.0.0, EOS VM (JIT or Optimized Compiler variants) can be used instead of WAVM. So WAVM has now been removed as a WASM runtime option from nodeos. - -### Deprecation notice reminder - -Please refer to the [Consolidated EOSIO Deprecations List](https://github.com/EOSIO/eos/issues/7597) for the currently active set of deprecation notices. - -## Other Changes - -- ([#7247](https://github.com/EOSIO/eos/pull/7247)) Change default log level from debug to info. - develop -- ([#7238](https://github.com/EOSIO/eos/pull/7238)) Remove unused cpack bits; these are not being used -- ([#7248](https://github.com/EOSIO/eos/pull/7248)) Pass env to build script installs -- ([#6913](https://github.com/EOSIO/eos/pull/6913)) Block log util#6884 -- ([#7249](https://github.com/EOSIO/eos/pull/7249)) Http configurable logging -- ([#7255](https://github.com/EOSIO/eos/pull/7255)) Move Test Metrics Code to Agent -- ([#7217](https://github.com/EOSIO/eos/pull/7217)) Created Universal Pipeline Configuration File -- ([#7264](https://github.com/EOSIO/eos/pull/7264)) use protocol-features-sync-nodes branch for LRT pipeline - develop -- ([#7207](https://github.com/EOSIO/eos/pull/7207)) Optimize mongodb plugin -- ([#7276](https://github.com/EOSIO/eos/pull/7276)) correct net_plugin's win32 check -- ([#7279](https://github.com/EOSIO/eos/pull/7279)) build unittests in c++17, not c++14 -- ([#7282](https://github.com/EOSIO/eos/pull/7282)) Fix cleos REX help - develop -- ([#7284](https://github.com/EOSIO/eos/pull/7284)) remove boost asio string_view workaround -- ([#7286](https://github.com/EOSIO/eos/pull/7286)) chainbase uniqueness violation fixes -- ([#7287](https://github.com/EOSIO/eos/pull/7287)) restrict range of error codes that contracts are allowed to emit -- ([#7278](https://github.com/EOSIO/eos/pull/7278)) simplify openssl setup in cmakelists -- ([#7288](https://github.com/EOSIO/eos/pull/7288)) Changes for Boost 1_70_0 -- ([#7285](https://github.com/EOSIO/eos/pull/7285)) Use of Mac Anka Fleet instead of iMac fleet -- ([#7293](https://github.com/EOSIO/eos/pull/7293)) Update EosioTester.cmake.in (develop) -- ([#7290](https://github.com/EOSIO/eos/pull/7290)) Block log util test -- ([#6845](https://github.com/EOSIO/eos/pull/6845)) Net plugin multithread -- ([#7295](https://github.com/EOSIO/eos/pull/7295)) Modify the pipeline control file for triggered builds -- ([#7305](https://github.com/EOSIO/eos/pull/7305)) Eliminating trigger to allow for community PR jobs to run again -- ([#7306](https://github.com/EOSIO/eos/pull/7306)) when unable to find mongo driver stop cmake -- ([#7309](https://github.com/EOSIO/eos/pull/7309)) add debug_mode to state history plugin - develop -- ([#7310](https://github.com/EOSIO/eos/pull/7310)) Switched to git checkout of commit + supporting forked repo -- ([#7291](https://github.com/EOSIO/eos/pull/7291)) add DB guard check for nodeos_under_min_avail_ram.py -- ([#7240](https://github.com/EOSIO/eos/pull/7240)) add signal tests -- ([#7315](https://github.com/EOSIO/eos/pull/7315)) Add REX balance info to cleos get account command -- ([#7304](https://github.com/EOSIO/eos/pull/7304)) transaction_metadata thread safety -- ([#7321](https://github.com/EOSIO/eos/pull/7321)) Only call init if system contract is loaded -- ([#7275](https://github.com/EOSIO/eos/pull/7275)) remove win32 from CMakeLists.txt -- ([#7322](https://github.com/EOSIO/eos/pull/7322)) Fix under min avail ram test -- ([#7327](https://github.com/EOSIO/eos/pull/7327)) fix block serialization order in fork_database::close - develop -- ([#7331](https://github.com/EOSIO/eos/pull/7331)) Use debug level logging for --verbose -- ([#7325](https://github.com/EOSIO/eos/pull/7325)) Look in both lib and lib64 for CMake modules when building EOSIO Tester -- ([#7337](https://github.com/EOSIO/eos/pull/7337)) correct signed mismatch warning in http_plugin -- ([#7348](https://github.com/EOSIO/eos/pull/7348)) Anka develop fix -- ([#7320](https://github.com/EOSIO/eos/pull/7320)) Add test for various chainbase objects which contain fields that require dynamic allocation -- ([#7350](https://github.com/EOSIO/eos/pull/7350)) correct signed mismatch warning in chain controller -- ([#7356](https://github.com/EOSIO/eos/pull/7356)) Fixes for Boost 1.70 to compile with our current CMake -- ([#7359](https://github.com/EOSIO/eos/pull/7359)) update to use boost 1.70 -- ([#7341](https://github.com/EOSIO/eos/pull/7341)) Add Version Check for Package Builder -- ([#7367](https://github.com/EOSIO/eos/pull/7367)) Fix exception # -- ([#7342](https://github.com/EOSIO/eos/pull/7342)) Update to appbase max priority on main thread -- ([#7336](https://github.com/EOSIO/eos/pull/7336)) (softfloat sync) clean up strict-aliasing rules warnings -- ([#7371](https://github.com/EOSIO/eos/pull/7371)) Adds configuration for replay test pipeline -- ([#7370](https://github.com/EOSIO/eos/pull/7370)) Fix `-b` flag for `cleos get table` subcommand -- ([#7369](https://github.com/EOSIO/eos/pull/7369)) Add `boost/asio/io_context.hpp` header to `transaction_metadata.hpp` for branch `develop` -- ([#7385](https://github.com/EOSIO/eos/pull/7385)) Ship: port #7383 and #7384 to develop -- ([#7377](https://github.com/EOSIO/eos/pull/7377)) Allow aliases of variants in ABI -- ([#7316](https://github.com/EOSIO/eos/pull/7316)) Explicit name -- ([#7389](https://github.com/EOSIO/eos/pull/7389)) Fix develop merge -- ([#7379](https://github.com/EOSIO/eos/pull/7379)) transaction deadline cleanup -- ([#7380](https://github.com/EOSIO/eos/pull/7380)) Producer incoming-transaction-queue-size-mb -- ([#7391](https://github.com/EOSIO/eos/pull/7391)) Allow for EOS clone to be a submodule -- ([#7390](https://github.com/EOSIO/eos/pull/7390)) port db_modes_test to python -- ([#7399](https://github.com/EOSIO/eos/pull/7399)) throw error if trying to create non R1 key on SE or YubiHSM wallet -- ([#7394](https://github.com/EOSIO/eos/pull/7394)) (fc sync) static_variant improvements & fix certificate trust when trust settings is empty -- ([#7405](https://github.com/EOSIO/eos/pull/7405)) Centralize EOSIO Pipeline -- ([#7401](https://github.com/EOSIO/eos/pull/7401)) state database versioning -- ([#7392](https://github.com/EOSIO/eos/pull/7392)) Trx blk connections -- ([#7433](https://github.com/EOSIO/eos/pull/7433)) use imported targets for boost & cleanup fc/appbase/chainbase standalone usage -- ([#7434](https://github.com/EOSIO/eos/pull/7434)) (chainbase) don’t keep file mapping active when in heap/locked mode -- ([#7432](https://github.com/EOSIO/eos/pull/7432)) No need to start keosd for cleos command which doesn't need keosd -- ([#7430](https://github.com/EOSIO/eos/pull/7430)) Add option for cleos sign subcommand to ask keosd for signing -- ([#7425](https://github.com/EOSIO/eos/pull/7425)) txn json to file -- ([#7440](https://github.com/EOSIO/eos/pull/7440)) Fix #7436 SIGSEGV - develop -- ([#7461](https://github.com/EOSIO/eos/pull/7461)) Name txn_test_gen threads -- ([#7442](https://github.com/EOSIO/eos/pull/7442)) Enhance cleos error message when parsing JSON argument -- ([#7451](https://github.com/EOSIO/eos/pull/7451)) set initial costs for expensive parallel unit tests -- ([#7366](https://github.com/EOSIO/eos/pull/7366)) BATS bash tests for build scripts + various other improvements and fixes -- ([#7467](https://github.com/EOSIO/eos/pull/7467)) Pipeline Configuration File Update -- ([#7476](https://github.com/EOSIO/eos/pull/7476)) Various improvements from pull/7458 -- ([#7488](https://github.com/EOSIO/eos/pull/7488)) Various BATS fixes to fix CI/CD -- ([#7489](https://github.com/EOSIO/eos/pull/7489)) Fix Incorrectly Resolved and Untested Merge Conflicts in Pipeline Configuration File -- ([#7474](https://github.com/EOSIO/eos/pull/7474)) fix copy_bin() for win32 builds -- ([#7478](https://github.com/EOSIO/eos/pull/7478)) remove stray SIGUSR1 -- ([#7475](https://github.com/EOSIO/eos/pull/7475)) guard unix socket support in http_plugin depending on platform support -- ([#7492](https://github.com/EOSIO/eos/pull/7492)) Enabled helpers for unpinned builds. -- ([#7482](https://github.com/EOSIO/eos/pull/7482)) Let delete-all-blocks option to only remove the contents instead of the directory itself -- ([#7502](https://github.com/EOSIO/eos/pull/7502)) [develop] Versioned images (prep for hashing) -- ([#7507](https://github.com/EOSIO/eos/pull/7507)) use create_directories in initialize_protocol_features - develop -- ([#7484](https://github.com/EOSIO/eos/pull/7484)) return zero exit status for nodeos version, help, fixed reversible, and extracted genesis -- ([#7468](https://github.com/EOSIO/eos/pull/7468)) Versioning library -- ([#7486](https://github.com/EOSIO/eos/pull/7486)) [develop] Ensure we're in repo root -- ([#7515](https://github.com/EOSIO/eos/pull/7515)) Custom path support for eosio installation. -- ([#7516](https://github.com/EOSIO/eos/pull/7516)) on supported platforms build with system clang by default -- ([#7519](https://github.com/EOSIO/eos/pull/7519)) [develop] Removed lrt from pipeline.jsonc -- ([#7525](https://github.com/EOSIO/eos/pull/7525)) [develop] Readlink quick fix and BATS test fixes -- ([#7532](https://github.com/EOSIO/eos/pull/7532)) [develop] BASE IMAGE Fixes -- ([#7535](https://github.com/EOSIO/eos/pull/7535)) Add -j option to print JSON format for cleos get currency balance. -- ([#7537](https://github.com/EOSIO/eos/pull/7537)) connection via listen needs to start in connecting mode -- ([#7497](https://github.com/EOSIO/eos/pull/7497)) properly add single quotes for parameter with spaces in logs output -- ([#7544](https://github.com/EOSIO/eos/pull/7544)) restore usage of devtoolset-8 on centos7 -- ([#7547](https://github.com/EOSIO/eos/pull/7547)) Sighup logging -- ([#7421](https://github.com/EOSIO/eos/pull/7421)) WebAuthn key and signature support -- ([#7572](https://github.com/EOSIO/eos/pull/7572)) [develop] Don't create mongo folders unless ENABLE_MONGO is true -- ([#7576](https://github.com/EOSIO/eos/pull/7576)) [develop] Better found/not found messages for clarity -- ([#7545](https://github.com/EOSIO/eos/pull/7545)) add nodiscard attribute to tester's push_action -- ([#7581](https://github.com/EOSIO/eos/pull/7581)) Update to fc with logger fix -- ([#7558](https://github.com/EOSIO/eos/pull/7558)) [develop] Ability to set *_DIR on CLI -- ([#7553](https://github.com/EOSIO/eos/pull/7553)) [develop] CMAKE version check before dependencies are installed -- ([#7584](https://github.com/EOSIO/eos/pull/7584)) fix hash<>::result_type deprecation spam -- ([#7588](https://github.com/EOSIO/eos/pull/7588)) [develop] Various BATS test fixes -- ([#7578](https://github.com/EOSIO/eos/pull/7578)) [develop] -i support for relative paths -- ([#7449](https://github.com/EOSIO/eos/pull/7449)) add accurate checktime timer for macOS -- ([#7591](https://github.com/EOSIO/eos/pull/7591)) [develop] SUDO_COMMAND -> NEW_SUDO_COMMAND (base fixed) -- ([#7594](https://github.com/EOSIO/eos/pull/7594)) [develop] Install location fix -- ([#7586](https://github.com/EOSIO/eos/pull/7586)) Modify transaction_ack to process bcast_transaction and rejected_transaction correctly -- ([#7585](https://github.com/EOSIO/eos/pull/7585)) Enhance cleos to enable new RPC send_transaction -- ([#7599](https://github.com/EOSIO/eos/pull/7599)) Use updated sync nodes for sync tests -- ([#7608](https://github.com/EOSIO/eos/pull/7608)) indicate in brew bottle mojave is required -- ([#7615](https://github.com/EOSIO/eos/pull/7615)) Change hardcoded currency symbol in testnet.template into a variable -- ([#7610](https://github.com/EOSIO/eos/pull/7610)) use explicit billing for unapplied and deferred transactions in tester - develop -- ([#7621](https://github.com/EOSIO/eos/pull/7621)) Call resolve on connection strand -- ([#7623](https://github.com/EOSIO/eos/pull/7623)) additional wasm unittests around max depth -- ([#7607](https://github.com/EOSIO/eos/pull/7607)) Improve nodeos make-index speeds -- ([#7598](https://github.com/EOSIO/eos/pull/7598)) Net plugin block id notification -- ([#7624](https://github.com/EOSIO/eos/pull/7624)) bios-boot-tutorial.py: bugfix, SYS hardcoded instead of using command… -- ([#7632](https://github.com/EOSIO/eos/pull/7632)) [develop] issues/7627: Install script missing ! -- ([#7634](https://github.com/EOSIO/eos/pull/7634)) fix fc::temp_directory usage in tester -- ([#7642](https://github.com/EOSIO/eos/pull/7642)) remove stale warning about dirty metadata -- ([#7640](https://github.com/EOSIO/eos/pull/7640)) fix win32 build of eosio-blocklog -- ([#7643](https://github.com/EOSIO/eos/pull/7643)) remove unused dlfcn.h include; troublesome for win32 -- ([#7645](https://github.com/EOSIO/eos/pull/7645)) add eosio-blocklog to base install component -- ([#7651](https://github.com/EOSIO/eos/pull/7651)) Port #7619 to develop -- ([#7652](https://github.com/EOSIO/eos/pull/7652)) remove raise() in keosd in favor of simple appbase quit (de-posix it) -- ([#7453](https://github.com/EOSIO/eos/pull/7453)) Refactor unapplied transaction queue -- ([#7657](https://github.com/EOSIO/eos/pull/7657)) (chainbase sync) print name of DB causing failure condition & win32 fixes -- ([#7663](https://github.com/EOSIO/eos/pull/7663)) Fix path error in cleos set code/abi -- ([#7633](https://github.com/EOSIO/eos/pull/7633)) Small optimization to move more trx processing off application thread -- ([#7662](https://github.com/EOSIO/eos/pull/7662)) fix fork resolve in special case -- ([#7667](https://github.com/EOSIO/eos/pull/7667)) fix 7600 double confirm after changing sign key -- ([#7625](https://github.com/EOSIO/eos/pull/7625)) Fix flaky tests - mainly net_plugin fixes -- ([#7672](https://github.com/EOSIO/eos/pull/7672)) Fix memory leak -- ([#7676](https://github.com/EOSIO/eos/pull/7676)) Remove unused code -- ([#7677](https://github.com/EOSIO/eos/pull/7677)) Commas go outside the quotes... -- ([#7675](https://github.com/EOSIO/eos/pull/7675)) wasm unit test with an imported function as start function -- ([#7678](https://github.com/EOSIO/eos/pull/7678)) Issue 3516 fix -- ([#7404](https://github.com/EOSIO/eos/pull/7404)) wtmsig block production -- ([#7686](https://github.com/EOSIO/eos/pull/7686)) Unapplied transaction queue performance -- ([#7685](https://github.com/EOSIO/eos/pull/7685)) Integration Test descriptions and timeout fix -- ([#7691](https://github.com/EOSIO/eos/pull/7691)) Fix nodeos 1.8.x to > 1.7.x peering issue (allowed-connection not equal to "any") -- ([#7250](https://github.com/EOSIO/eos/pull/7250)) wabt: reduce redundant memset -- ([#7702](https://github.com/EOSIO/eos/pull/7702)) fix producer_plugin watermark tracking - develop -- ([#7477](https://github.com/EOSIO/eos/pull/7477)) Fix abi_serializer to encode optional non-built_in types -- ([#7703](https://github.com/EOSIO/eos/pull/7703)) return flat_multimap from transaction::validate_and_extract_extensions -- ([#7720](https://github.com/EOSIO/eos/pull/7720)) Fix bug to make sed -i work properly on Mac -- ([#7716](https://github.com/EOSIO/eos/pull/7716)) [TRAVIS POC] develop Support passing in JOBS for docker/kube multi-tenancy -- ([#7707](https://github.com/EOSIO/eos/pull/7707)) add softfloat only injection mode -- ([#7725](https://github.com/EOSIO/eos/pull/7725)) Fix increment in test -- ([#7729](https://github.com/EOSIO/eos/pull/7729)) Fix db_modes_test -- ([#7734](https://github.com/EOSIO/eos/pull/7734)) Fix db exhaustion -- ([#7487](https://github.com/EOSIO/eos/pull/7487)) Enable extended_asset to be encoded from array -- ([#7736](https://github.com/EOSIO/eos/pull/7736)) unit test ensuring that OOB table init allowed on set code; fails on action -- ([#7744](https://github.com/EOSIO/eos/pull/7744)) (appbase) update to get non-option fix & unique_ptr tweak -- ([#7746](https://github.com/EOSIO/eos/pull/7746)) (chainbase sync) fix build with boost 1.71 -- ([#7721](https://github.com/EOSIO/eos/pull/7721)) Improve signature recovery -- ([#7757](https://github.com/EOSIO/eos/pull/7757)) remove stale license headers -- ([#7756](https://github.com/EOSIO/eos/pull/7756)) block_log performance improvement, and misc. -- ([#7654](https://github.com/EOSIO/eos/pull/7654)) exclusively use timer for checktime -- ([#7763](https://github.com/EOSIO/eos/pull/7763)) Use fc::cfile instead of std::fstream for state_history -- ([#7770](https://github.com/EOSIO/eos/pull/7770)) Net plugin sync fix -- ([#7717](https://github.com/EOSIO/eos/pull/7717)) Support for v2 snapshots with pending producer schedules -- ([#7795](https://github.com/EOSIO/eos/pull/7795)) Hardcode initial eosio ABI: #7794 -- ([#7792](https://github.com/EOSIO/eos/pull/7792)) Restore default logging if logging.json is removed when SIGHUP. -- ([#7791](https://github.com/EOSIO/eos/pull/7791)) Producer plugin -- ([#7786](https://github.com/EOSIO/eos/pull/7786)) Remove redundant work from net plugin -- ([#7785](https://github.com/EOSIO/eos/pull/7785)) Optimize block log usage -- ([#7812](https://github.com/EOSIO/eos/pull/7812)) Add IMPORTANT file and update README - develop -- ([#7820](https://github.com/EOSIO/eos/pull/7820)) rename IMPORTANT to IMPORTANT.md - develop -- ([#7809](https://github.com/EOSIO/eos/pull/7809)) Correct cpu_usage calculation when more than one signature -- ([#7803](https://github.com/EOSIO/eos/pull/7803)) callback support for checktime timer expiry -- ([#7838](https://github.com/EOSIO/eos/pull/7838)) Bandwidth - develop -- ([#7845](https://github.com/EOSIO/eos/pull/7845)) Deprecate network version match - develop -- ([#7700](https://github.com/EOSIO/eos/pull/7700)) [develop] Travis CI + Buildkite 3.0 -- ([#7825](https://github.com/EOSIO/eos/pull/7825)) apply_block optimization -- ([#7849](https://github.com/EOSIO/eos/pull/7849)) Increase Contracts Builder Timeout + Fix $SKIP_MAC -- ([#7854](https://github.com/EOSIO/eos/pull/7854)) Net plugin sync -- ([#7860](https://github.com/EOSIO/eos/pull/7860)) [develop] Ensure release flag is added to all builds. -- ([#7873](https://github.com/EOSIO/eos/pull/7873)) [develop] Mac Builder Boost Fix -- ([#7868](https://github.com/EOSIO/eos/pull/7868)) cleos get actions -- ([#7774](https://github.com/EOSIO/eos/pull/7774)) update WAVM to be compatible with LLVM 7 through 9 -- ([#7864](https://github.com/EOSIO/eos/pull/7864)) Add output of build info on nodeos startup -- ([#7881](https://github.com/EOSIO/eos/pull/7881)) [develop] Ensure Artfacts Upload on Failed Tests -- ([#7886](https://github.com/EOSIO/eos/pull/7886)) promote read-only disablement log from net_plugin to warn level -- ([#7887](https://github.com/EOSIO/eos/pull/7887)) print unix socket path when there is an error starting unix socket server -- ([#7889](https://github.com/EOSIO/eos/pull/7889)) Update docker builder tag -- ([#7891](https://github.com/EOSIO/eos/pull/7891)) Fix exit crash - develop -- ([#7877](https://github.com/EOSIO/eos/pull/7877)) Create Release Build Test -- ([#7883](https://github.com/EOSIO/eos/pull/7883)) Add Support for eosio-test-stability Pipeline -- ([#7903](https://github.com/EOSIO/eos/pull/7903)) adds support for builder priority queues -- ([#7853](https://github.com/EOSIO/eos/pull/7853)) change behavior of recover_key to better support variable length keys -- ([#7901](https://github.com/EOSIO/eos/pull/7901)) [develop] Add Trigger for LRTs and Multiversion Tests Post PR -- ([#7914](https://github.com/EOSIO/eos/pull/7914)) [Develop] Forked PR fix -- ([#7923](https://github.com/EOSIO/eos/pull/7923)) return error when attempting to remove key from YubiHSM wallet -- ([#7910](https://github.com/EOSIO/eos/pull/7910)) [Develop] Updated anka plugin, added failover for registries, and added sleep fix for git clone/networking bug -- ([#7926](https://github.com/EOSIO/eos/pull/7926)) Better error check in test -- ([#7919](https://github.com/EOSIO/eos/pull/7919)) decouple wavm runtime from being required & initial support for platform specific wasm runtimes -- ([#7927](https://github.com/EOSIO/eos/pull/7927)) Fix Release Build Type for macOS on Travis CI -- ([#7931](https://github.com/EOSIO/eos/pull/7931)) Fix intermittent crash on exit when port already in use - develop -- ([#7930](https://github.com/EOSIO/eos/pull/7930)) use -fdiagnostics-color=always even for clang -- ([#7933](https://github.com/EOSIO/eos/pull/7933)) [Develop] Support all BK/Travis cases in Submodule Regression Script -- ([#7943](https://github.com/EOSIO/eos/pull/7943)) Change eosio-launcher enable-gelf-logging argument default to false. -- ([#7946](https://github.com/EOSIO/eos/pull/7946)) Forked chain test error statement - develop -- ([#7948](https://github.com/EOSIO/eos/pull/7948)) net_plugin correctly handle unknown_block_exception - develop -- ([#7954](https://github.com/EOSIO/eos/pull/7954)) remove bad semicolon (in unused but compiled code) -- ([#7952](https://github.com/EOSIO/eos/pull/7952)) Unable to Create Block Log Index #7865 -- ([#7953](https://github.com/EOSIO/eos/pull/7953)) Refactor producer plugin start_block - develop -- ([#7841](https://github.com/EOSIO/eos/pull/7841)) 7646 chain id in blog -- ([#7958](https://github.com/EOSIO/eos/pull/7958)) [develop] Fix Mac builds on Travis -- ([#7962](https://github.com/EOSIO/eos/pull/7962)) set immutable chain_id during construction of controller -- ([#7957](https://github.com/EOSIO/eos/pull/7957)) Upgrade to Boost 1.71.0 -- ([#7971](https://github.com/EOSIO/eos/pull/7971)) Net plugin unexpected block - develop -- ([#7967](https://github.com/EOSIO/eos/pull/7967)) support unix socket HTTP server for nodeos -- ([#7947](https://github.com/EOSIO/eos/pull/7947)) Function body code size test -- ([#7955](https://github.com/EOSIO/eos/pull/7955)) EOSIO WASM Spec tests -- ([#7978](https://github.com/EOSIO/eos/pull/7978)) use the LLVM 7 library provided by SCL on CentOS7 -- ([#7983](https://github.com/EOSIO/eos/pull/7983)) port consolidated security fixes for 1.8.4 to develop; add unit tests associated with consolidated security fixes for 1.8.1 -- ([#7986](https://github.com/EOSIO/eos/pull/7986)) Remove unnecessary comment -- ([#7985](https://github.com/EOSIO/eos/pull/7985)) more bug fixes with chain_id in state changes -- ([#7974](https://github.com/EOSIO/eos/pull/7974)) Experimental/wb2 jit -- ([#7989](https://github.com/EOSIO/eos/pull/7989)) Correct designator order for field of get_table_rows_params -- ([#7992](https://github.com/EOSIO/eos/pull/7992)) move wasm_allocator from wasm_interface to controller -- ([#7995](https://github.com/EOSIO/eos/pull/7995)) new timeout to handle when two jobs on the same host are maxing their… -- ([#7993](https://github.com/EOSIO/eos/pull/7993)) update eos-vm to latest develop, fix issues with instantiation limit … -- ([#7991](https://github.com/EOSIO/eos/pull/7991)) missing block log chain id unit tests -- ([#8001](https://github.com/EOSIO/eos/pull/8001)) Net plugin trx progress - develop -- ([#8003](https://github.com/EOSIO/eos/pull/8003)) update eos-vm ref -- ([#7988](https://github.com/EOSIO/eos/pull/7988)) Net plugin version match -- ([#8004](https://github.com/EOSIO/eos/pull/8004)) bump version -- ([#7975](https://github.com/EOSIO/eos/pull/7975)) EOS-VM Optimized Compiler -- ([#8007](https://github.com/EOSIO/eos/pull/8007)) disallow WAVM with EOS-VM OC -- ([#8010](https://github.com/EOSIO/eos/pull/8010)) Change log level of index write -- ([#8009](https://github.com/EOSIO/eos/pull/8009)) pending incoming order on subjective failure -- ([#8013](https://github.com/EOSIO/eos/pull/8013)) ensure eos-vm-oc headers get installed -- ([#8008](https://github.com/EOSIO/eos/pull/8008)) Increase stability of nodeos_under_min_avail_ram.py - develop -- ([#8015](https://github.com/EOSIO/eos/pull/8015)) two fixes for eosio tester cmake modules -- ([#8019](https://github.com/EOSIO/eos/pull/8019)) [Develop] Change submodule script to see stderr for git commands -- ([#8014](https://github.com/EOSIO/eos/pull/8014)) Retain persisted trx until expired on speculative nodes -- ([#8024](https://github.com/EOSIO/eos/pull/8024)) Add optional ability to disable WASM Spec Tests -- ([#8023](https://github.com/EOSIO/eos/pull/8023)) Make subjective_cpu_leeway a config option -- ([#8025](https://github.com/EOSIO/eos/pull/8025)) Fix build script LLVM symlinking -- ([#8026](https://github.com/EOSIO/eos/pull/8026)) update EOS VM Optimized Compiler naming convention -- ([#8012](https://github.com/EOSIO/eos/pull/8012)) 7939 trim block log v3 support -- ([#8029](https://github.com/EOSIO/eos/pull/8029)) update eos-vm ref and install eos-vm license -- ([#8033](https://github.com/EOSIO/eos/pull/8033)) net_plugin better error for unknown block -- ([#8034](https://github.com/EOSIO/eos/pull/8034)) EOS VM OC license updates -- ([#8042](https://github.com/EOSIO/eos/pull/8042)) [2.0.x] dockerhub | eosio/producer -> eosio/ci -- ([#8050](https://github.com/EOSIO/eos/pull/8050)) Add greylist limit - v2.0.x -- ([#8060](https://github.com/EOSIO/eos/pull/8060)) #8054: fix commas in ship abi -- ([#8072](https://github.com/EOSIO/eos/pull/8072)) nodeos & keosd version reporting - 2.0 -- ([#8071](https://github.com/EOSIO/eos/pull/8071)) Update cleos to support new producer schedule - 2.0 -- ([#8070](https://github.com/EOSIO/eos/pull/8070)) don't rebuild llvm unnecessarily during pinned builds - 2.0 -- ([#8074](https://github.com/EOSIO/eos/pull/8074)) [2.0.x] Upgrade mac anka template to 10.14.6 -- ([#8076](https://github.com/EOSIO/eos/pull/8076)) Handle cases where version_* not specified in CMakeLists.txt - 2.0 -- ([#8088](https://github.com/EOSIO/eos/pull/8088)) [2.0.x] Linux build fleet update -- ([#8091](https://github.com/EOSIO/eos/pull/8091)) report block extensions_type contents in RPC and eosio-blocklog tool - 2.0 -- ([#8105](https://github.com/EOSIO/eos/pull/8105)) Modify --print-default-config to exit with success - 2.0 -- ([#8113](https://github.com/EOSIO/eos/pull/8113)) [2.0.x] WASM Spec Test Step in CI -- ([#8114](https://github.com/EOSIO/eos/pull/8114)) [2.0.x] Mac OSX steps need a min of 1 hour -- ([#8127](https://github.com/EOSIO/eos/pull/8127)) [2.0.x] Move the ensure step into the build step, eliminating the need for templaters -- ([#8144](https://github.com/EOSIO/eos/pull/8144)) fix pinned builds on fresh macOS install - 2.0 -- ([#8149](https://github.com/EOSIO/eos/pull/8149)) [2.0.x] CI platform directories -- ([#8155](https://github.com/EOSIO/eos/pull/8155)) Post State history callback as medium priority - 2.0 -- ([#8173](https://github.com/EOSIO/eos/pull/8173)) ensure GMP is always dynamically linked - 2.0 -- ([#8175](https://github.com/EOSIO/eos/pull/8175)) [2.0.x] Unpinned and WASM test fixes -- ([#8168](https://github.com/EOSIO/eos/pull/8168)) add harden flags to cicd & pinned builds - 2.0 -- ([#8180](https://github.com/EOSIO/eos/pull/8180)) [2.0.x] 10 second sleep to address heavy usage wait-network bug in Anka -- ([#8192](https://github.com/EOSIO/eos/pull/8192)) Reduce logging - 2.0 -- ([#8367](https://github.com/EOSIO/eos/pull/8367)) Add Sync from Genesis Test -- ([#8363](https://github.com/EOSIO/eos/pull/8363)) Fix linking OpenSSL (branch `release/2.0.x`) -- ([#8383](https://github.com/EOSIO/eos/pull/8383)) Escape BUILDKITE_COMMIT to generate tag properly -- ([#8385](https://github.com/EOSIO/eos/pull/8385)) Propagate exceptions out push_block - 2.0 -- ([#8391](https://github.com/EOSIO/eos/pull/8391)) Add eosio-resume-from-state Test -- ([#8393](https://github.com/EOSIO/eos/pull/8393)) Make multiversion protocol test conditional. -- ([#8402](https://github.com/EOSIO/eos/pull/8402)) fix EOS VM OC monitor thread name - 2.0 -- ([#8406](https://github.com/EOSIO/eos/pull/8406)) [2.0.x] Modified Amazon and Centos to use yum install ccache -- ([#8414](https://github.com/EOSIO/eos/pull/8414)) Add better logging of exceptions in emit - 2.0 -- ([#8328](https://github.com/EOSIO/eos/pull/8328)) Fix bios boot python script due to 2.0.x changes -- ([#8293](https://github.com/EOSIO/eos/pull/8293)) Add nodeos/cleos/keosd docs from develop, update README -- ([#8425](https://github.com/EOSIO/eos/pull/8425)) fix discovery of openssl in tester cmake when OPENSSL_ROOT_DIR not set - 2.0 - -## Thanks! - -Special thanks to the community contributors that submitted patches for this release: -- @UMU618 -- @conr2d -- @YordanPavlov -- @baegjae -- @olexiybuyanskyy -- @spartucus -- @rdewilder diff --git a/docs/30_release-notes/99_v2.1.0-rc1.md b/docs/30_release-notes/99_v2.1.0-rc1.md new file mode 100644 index 00000000000..65c40eca818 --- /dev/null +++ b/docs/30_release-notes/99_v2.1.0-rc1.md @@ -0,0 +1,607 @@ +--- +content_title: EOSIO v2.1.0-rc1 Release Notes +link_text: v2.1.0-rc1 +--- + +This is a ***RELEASE CANDIDATE*** for version 2.1.0. + +While EOSIO has always been innovative and highly-performant, this release focuses on making it easier to build large-scale applications on the platform, and to maintain them once they’re deployed. It is a reflection of our commitment to abstract away some of the complexities of blockchain development and make it approachable to a broader audience. + +EOSIO 2.1.0-rc1 marks the first time we’re releasing a feature that is specifically intended for private blockchains only, with the the ability to remove Context-Free Data. This feature will provide a way for private blockchain administrators to delete a specifically designated section of data, without compromising the integrity of the chain. + +The EOSIO 2.1.0-rc1 also includes additional features that optimize blockchain data storage, simplify table management, and provide clustering options for system administrators. + +We encourage developers to test the additional features in the EOSIO 2.1.0-rc1, and provide us with feedback. If you would like to offer feedback on the release candidate of EOSIO 2.1.0 and work more closely with our team to improve EOSIO for developers, you can contact our developer relations team at developers@block.one. + +## Changes + +### Action Return Values ([#8327](https://github.com/EOSIO/eos/pull/8327)) +New protocol feature: `ACTION_RETURN_VALUE`. When activated, this feature provides a way to get return values which are strongly committed to in block headers from actions into external processes without having to rely on get_table or using the debug console via print statements. This allows smart contract developers to be able to process the return value from an action directly; further streamlining the smart contract development process. An example can be seen [here.](https://github.com/EOSIO/return-values-example-app) + +### Configurable WASM Limits ([#8360](https://github.com/EOSIO/eos/pull/8360)) +New protocol feature: `CONFIGURABLE_WASM_LIMITS`. When activated, this feature allows privileged contracts to set the constraints on WebAssembly code. + +### Extensible Blockchain Parameters ([#9402](https://github.com/EOSIO/eos/pull/9402)) +The basic means of manipulating consensus parameters for an EOSIO blockchain has been a pair of intrinsic functions: `get_blockchain_parameters_packed` and `set_blockchain_parameters_packed`. These intrinsics are tied to a specific and inflexible definition of blockchain parameters and include no convenient means to _version_ the set of parameters; which is an inconvenience to add/remove/modify in future consensus upgrades. + +To alleviate this, Nodeos now has a new protocol feature: `BLOCKCHAIN_PARAMETERS`. When activated, this protocol feature is intended to eventually supplant the existing intrinsics and provide greater flexibility for future consensus upgrades. When activated it will allow contracts to link to the new intrinsics. + +### Health Logging For Nodeos running The State History Plugin ([#9208](https://github.com/EOSIO/eos/pull/9208)) ([#9239](https://github.com/EOSIO/eos/pull/9239)) ([#9277](https://github.com/EOSIO/eos/pull/9277)) +Nodeos now has added support for a separate logger to the state history plugin and add some additional logging messages for receiving requests and sending replies. In addition, the trace and chain state log can now be split in the state history plugin as well. + +### Instrumentation Support for Nodeos ([#9631](https://github.com/EOSIO/eos/pull/9631)) +Nodeos now supports integration with Zipkin, an open source distributed tracing system. This will enable system administrators to optimize Nodeos execution for performance-critical applications. + +### Key Value Tables ([#8223](https://github.com/EOSIO/eos/pull/8223), [#9298](https://github.com/EOSIO/eos/pull/9298)) +New protocol feature: `KV_DATABASE`. When activated, this feature provides a Key Value API. This new API is a more flexible, simplified way for developers to create and search on-chain tables. Developers can also modify the table structure after it has been created, which is currently impossible with multi-index tables. + +Developers will also be able to split up tables they have already written. An example of this is in the case where the developer has a table that stores a user’s first and last name along with other information. The developer could now decide to split the original table into two separate tables, one containing the first names and one containing the last names. + +As with the existing db api, contracts can flexibly specify which authorizing account provides the RAM resources for this data. + +An example can be seen [here.](https://github.com/EOSIO/key-value-example-app) You can follow the instructions [here](https://github.com/EOSIO/eos/tree/develop/contracts/enable-kv) to quickly create a test chain with Key Value support. + +### Prune Context-Free Data ([#9061](https://github.com/EOSIO/eos/pull/9061)) +From inception, EOSIO has supported the concept of Context-Free Data, or data that may be removed without affecting the integrity of the chain. This release enables administrators to designate specific data as Context-Free and subsequently remove, or prune, that data from the blockchain while maintaining system stability. + +Once this data has been pruned, full validation is no longer possible, only light validation, which requires implicit trust in the block producers. Due to this factor, the Prune Context-Free Data feature is only suitable for a private blockchain as part a larger privacy, security, or regulatory compliance solution. + +### Support For Ubuntu 20.04, CentOS 7.x, and CentOS 8 ([#9332](https://github.com/EOSIO/eos/pull/9332)) ([#9475](https://github.com/EOSIO/eos/pull/9475)) +EOSIO now supports Ubuntu 20.04, CentOS 7.x, and CentOS 8, in addition to previous releases supporting Amazon Linux 2, CentOS 7, Ubuntu 16.04, Ubuntu 18.04 and MacOS 10.14 (Mojave). + +### Reimplement Chainbase Using Intrusive Instead of multi_index ([#58](https://github.com/EOSIO/chainbase/pull/58)) +Nodoes now features an upgraded version of chainbase using intrusive instead of multi_index. This makes chainbase more performant and features per container memory pools, full exception safety, lighter weight representation of the undo stack, and avl trees instead of rb trees. + +### [Developer Preview] Blockvault ([#9705](https://github.com/EOSIO/eos/pull/9705)) +Nodeos now supports clustering for the block producer node, enabling blockchain administrators to implement industry standard disaster recovery architectures. Two or more nodes may be deployed as a single logical producer. If the primary node goes down, a system properly configured to leverage this solution can attain similar data recovery guarantees to that of industry leading database and cloud services, with minimal service disruption. + +While this feature increases resiliency for block production on public networks, it also provides particular value for private chains running with a single logical producer. Single-producer chains can use it to provide immediate finality with tools to mitigate the risk of a single point of failure. + +To use this feature, `nodeos` must be configured as a producer with the appropriate `--block-vault-backend` option specified. For example: + +``` +nodeos --plugin eosio::producer_plugin --producer-name myproducera --plugin eosio::blockvault_client_plugin --block-vault-backend postgresql://user:password@mycompany.com +``` + +For more information on using this feature please see the `README.md` file in directory `~/eos/plugins/blockvault_client_plugin/README.md`. + +This feature is being released as a "developer preview" and is not yet ready for production usage. We look forward to community feedback to further develop and harden this feature. + +### [Developer Preview] RocksDB Storage for DB and Key Value APIs ([#9340](https://github.com/EOSIO/eos/pull/9340)) ([#9529](https://github.com/EOSIO/eos/pull/9529)) +RocksDB is now supported as a storage option behind either the DB or Key Value APIs. This gives blockchain system administrators the flexibility to choose between RAM or RocksDB to optimize Nodeos performance for their workloads. + +To use this feature, `nodeos` must specify which backing store to use by passing the flag `--backing-store=rocksdb`. + +For more information on using this feature please see the `10_how-to-configure-state-storage.md` file in directory `~/eos/docs/01_nodeos/02_usage/60_how-to-guides/10_how-to-configure-state-storage.md`. + +This feature is being released as a "developer preview" and is not yet ready for production usage. We look forward to community feedback to further develop and harden this feature. + +## Known Issues +A known issue exists with accessing the right version of libpq.so on Centos 7.x, Amazon Linux 2, and Ubuntu 16.04 when running with the prebuilt binaries attached to the v2.1.0-rc1 release notes in Github (binaries located at the bottom of this page). On those platforms please build EOSIO from source using the provided `~/eos/scripts/eosio_build.sh` script using the instructions provided [here](https://developers.eos.io/manuals/eos/latest/install/build-from-source/shell-scripts/index) to overcome the issue (you will need to perform a `git checkout v2.1.0-rc1` followed by a `git submodule update --init --recursive` before running the script) + +## Deprecation and Removal Notices +- ([#8498](https://github.com/EOSIO/eos/pull/8498)) Remove new block id notify feature - develop +- ([#9014](https://github.com/EOSIO/eos/pull/9014)) Remove mongo_db_plugin +- ([#9701](https://github.com/EOSIO/eos/pull/9701)) remove long disabled faucet_testnet_plugin + +## Upgrading From previous versions of EOSIO + +### Upgrading From v2.0.x + +Node operators running version v2.0.x should be able to upgrade to v2.1.0-rc1 using a snapshot. In addition, moving from a chainbase-backed node to a RocksDB-backed node or the reverse will also require a snapshot to migrate. + +## Other Changes +- ([#7973](https://github.com/EOSIO/eos/pull/7973)) Add a unit test for the write order for aliased intrinsic arguments. +- ([#8039](https://github.com/EOSIO/eos/pull/8039)) [Develop] dockerhub | eosio/producer -> eosio/ci +- ([#8043](https://github.com/EOSIO/eos/pull/8043)) Refactor incoming trx handling +- ([#8044](https://github.com/EOSIO/eos/pull/8044)) Add greylist limit - develop +- ([#8046](https://github.com/EOSIO/eos/pull/8046)) #7658: modified code to handle new db_runtime_exception +- ([#8047](https://github.com/EOSIO/eos/pull/8047)) remove WAVM runtime +- ([#8049](https://github.com/EOSIO/eos/pull/8049)) Update cleos to support new producer schedule - develop +- ([#8053](https://github.com/EOSIO/eos/pull/8053)) don't rebuild llvm unnecessarily during pinned builds +- ([#8056](https://github.com/EOSIO/eos/pull/8056)) #7671 added checks for irreversible mode +- ([#8057](https://github.com/EOSIO/eos/pull/8057)) [Develop] Upgrade mac anka template to 10.14.6 +- ([#8062](https://github.com/EOSIO/eos/pull/8062)) nodeos & keosd version reporting +- ([#8073](https://github.com/EOSIO/eos/pull/8073)) disable terminfo usage on pinned llvm builds +- ([#8075](https://github.com/EOSIO/eos/pull/8075)) Handle cases where version_* not specified in CMakeLists.txt - develop +- ([#8077](https://github.com/EOSIO/eos/pull/8077)) Use BOOST_CHECK_EQUAL instead of BOOST_REQUIRE_EQUAL. +- ([#8082](https://github.com/EOSIO/eos/pull/8082)) report block extensions_type contents in RPC and eosio-blocklog tool - develop +- ([#8085](https://github.com/EOSIO/eos/pull/8085)) Net plugin remove read delays - develop +- ([#8089](https://github.com/EOSIO/eos/pull/8089)) [develop] Linux build fleet update +- ([#8094](https://github.com/EOSIO/eos/pull/8094)) net_plugin remove sync w/peer check - develop +- ([#8104](https://github.com/EOSIO/eos/pull/8104)) Modify --print-default-config to exit with success - develop +- ([#8106](https://github.com/EOSIO/eos/pull/8106)) Port PR #8060 to develop: fix commas in ship ABI +- ([#8107](https://github.com/EOSIO/eos/pull/8107)) [develop] WASM Spec Test Step in CI +- ([#8109](https://github.com/EOSIO/eos/pull/8109)) [Develop] Mac OSX steps need a min of 1 hour +- ([#8115](https://github.com/EOSIO/eos/pull/8115)) remove lingering wavm runtime file that escaped the first purge +- ([#8118](https://github.com/EOSIO/eos/pull/8118)) remove gettext/libintl dependency +- ([#8119](https://github.com/EOSIO/eos/pull/8119)) Net plugin sync fix - develop +- ([#8121](https://github.com/EOSIO/eos/pull/8121)) [Develop] Move the ensure step into the build step, eliminating the need for templaters +- ([#8130](https://github.com/EOSIO/eos/pull/8130)) #8129 - Fix spelling error in cleos/main.cpp +- ([#8131](https://github.com/EOSIO/eos/pull/8131)) Normalized capitalization in cleos/main.cpp +- ([#8132](https://github.com/EOSIO/eos/pull/8132)) [Develop] CI/CD support for Catalina +- ([#8135](https://github.com/EOSIO/eos/pull/8135)) [develop] CI platform directories +- ([#8136](https://github.com/EOSIO/eos/pull/8136)) explicitly link to zlib when compiling executables using the add_eosio_test_executable macro +- ([#8140](https://github.com/EOSIO/eos/pull/8140)) Post State history callback as medium priority - develop +- ([#8142](https://github.com/EOSIO/eos/pull/8142)) Net plugin sync priority +- ([#8143](https://github.com/EOSIO/eos/pull/8143)) fix pinned builds on fresh macOS install +- ([#8146](https://github.com/EOSIO/eos/pull/8146)) Update fc +- ([#8147](https://github.com/EOSIO/eos/pull/8147)) Optimize push_transaction +- ([#8151](https://github.com/EOSIO/eos/pull/8151)) Debian Package: Make sure root is owner/group when building dpkg. +- ([#8158](https://github.com/EOSIO/eos/pull/8158)) transactions in progress +- ([#8165](https://github.com/EOSIO/eos/pull/8165)) [Develop] Prevent buildkite clone to speedup pipeline +- ([#8166](https://github.com/EOSIO/eos/pull/8166)) Remove references to smart_ref. +- ([#8167](https://github.com/EOSIO/eos/pull/8167)) add harden flags to cicd & pinned builds +- ([#8172](https://github.com/EOSIO/eos/pull/8172)) [develop] Unpinned and WASM test fixes +- ([#8177](https://github.com/EOSIO/eos/pull/8177)) sync fc to pick up gmp fix & boost deque support +- ([#8178](https://github.com/EOSIO/eos/pull/8178)) [Develop] 10 second sleep to address heavy usage wait-network bug in Anka +- ([#8184](https://github.com/EOSIO/eos/pull/8184)) make DISABLE_WASM_SPEC_TESTS an option so it's visible from the GUI +- ([#8186](https://github.com/EOSIO/eos/pull/8186)) Update fc for EOSIO/fc#121 and EOSIO/fc#123 +- ([#8193](https://github.com/EOSIO/eos/pull/8193)) Reduce logging - develop +- ([#8194](https://github.com/EOSIO/eos/pull/8194)) Fixed under min available test to not count failed attempts as actual sends +- ([#8196](https://github.com/EOSIO/eos/pull/8196)) Consolidated Fixes for develop +- ([#8198](https://github.com/EOSIO/eos/pull/8198)) State History Plugin Integration Test +- ([#8208](https://github.com/EOSIO/eos/pull/8208)) eliminate gperftools copy paste +- ([#8209](https://github.com/EOSIO/eos/pull/8209)) stop setting CXX_FLAGS with both C & CXX flags +- ([#8217](https://github.com/EOSIO/eos/pull/8217)) Update chainbase to support Boost 1.67. +- ([#8218](https://github.com/EOSIO/eos/pull/8218)) Add option to provide transaction signature keys to cleos +- ([#8220](https://github.com/EOSIO/eos/pull/8220)) Add terminate-at-block option to nodeos. +- ([#8222](https://github.com/EOSIO/eos/pull/8222)) Many Transaction Long Running Test +- ([#8223](https://github.com/EOSIO/eos/pull/8223)) kv database +- ([#8231](https://github.com/EOSIO/eos/pull/8231)) return more from producer_plugin's get_runtime_options() +- ([#8232](https://github.com/EOSIO/eos/pull/8232)) Create integration test for sending copies of the same transaction into the network +- ([#8234](https://github.com/EOSIO/eos/pull/8234)) chainbase sync to pick up DB shrink fix while in heap mode +- ([#8245](https://github.com/EOSIO/eos/pull/8245)) [Develop] explictly use openssl 1.1 via brew on macos +- ([#8250](https://github.com/EOSIO/eos/pull/8250)) Spelling correction +- ([#8251](https://github.com/EOSIO/eos/pull/8251)) debug level logging for launcher service +- ([#8254](https://github.com/EOSIO/eos/pull/8254)) Replace hard coding system_account_name +- ([#8269](https://github.com/EOSIO/eos/pull/8269)) Remove Unused Variable +- ([#8274](https://github.com/EOSIO/eos/pull/8274)) [develop] Update CentOS version for CI. +- ([#8276](https://github.com/EOSIO/eos/pull/8276)) Net plugin sync - develop +- ([#8277](https://github.com/EOSIO/eos/pull/8277)) [develop] Travis updates. +- ([#8281](https://github.com/EOSIO/eos/pull/8281)) Net plugin handshake +- ([#8291](https://github.com/EOSIO/eos/pull/8291)) Exit irreversible mode test when failure occurrs +- ([#8299](https://github.com/EOSIO/eos/pull/8299)) net_plugin boost asio error handling +- ([#8300](https://github.com/EOSIO/eos/pull/8300)) net_plugin lib sync - develop +- ([#8304](https://github.com/EOSIO/eos/pull/8304)) net_plugin thread protection peer logging variables - develop +- ([#8306](https://github.com/EOSIO/eos/pull/8306)) Extend shutdown allowed time in under min available resources test +- ([#8312](https://github.com/EOSIO/eos/pull/8312)) Fix race in message_buffer and move message_buffer_tests to fc. - develop +- ([#8313](https://github.com/EOSIO/eos/pull/8313)) reset the new handler (develop) +- ([#8317](https://github.com/EOSIO/eos/pull/8317)) net_plugin speed up shutdown +- ([#8321](https://github.com/EOSIO/eos/pull/8321)) [develop] Retries and Contract Builders for Tags +- ([#8336](https://github.com/EOSIO/eos/pull/8336)) increase tester state size - develop +- ([#8339](https://github.com/EOSIO/eos/pull/8339)) Removing BATS tests +- ([#8340](https://github.com/EOSIO/eos/pull/8340)) [develop] Modification to trigger LRTs and Multiver on any protected branch that is not a scheduled run +- ([#8345](https://github.com/EOSIO/eos/pull/8345)) Remove superfluous quotes from default agent name string. +- ([#8349](https://github.com/EOSIO/eos/pull/8349)) Consolidated Security Fixes for Develop +- ([#8358](https://github.com/EOSIO/eos/pull/8358)) Add Sync from Genesis Test +- ([#8361](https://github.com/EOSIO/eos/pull/8361)) Make multiversion protocol test conditional. +- ([#8364](https://github.com/EOSIO/eos/pull/8364)) Fix linking OpenSSL (branch `develop`) +- ([#8374](https://github.com/EOSIO/eos/pull/8374)) CMAKE 3.16.2 +- ([#8382](https://github.com/EOSIO/eos/pull/8382)) Fix for NVM install +- ([#8387](https://github.com/EOSIO/eos/pull/8387)) Propagate exceptions out push_block - develop +- ([#8390](https://github.com/EOSIO/eos/pull/8390)) Add eosio-resume-from-state Test +- ([#8398](https://github.com/EOSIO/eos/pull/8398)) Net plugin sync check - develop +- ([#8401](https://github.com/EOSIO/eos/pull/8401)) fix EOS VM OC monitor thread name +- ([#8404](https://github.com/EOSIO/eos/pull/8404)) Revert: Debian Package: Make sure root is owner/group when building dpkg +- ([#8405](https://github.com/EOSIO/eos/pull/8405)) [develop] Modified Amazon and Centos to use yum install ccache +- ([#8408](https://github.com/EOSIO/eos/pull/8408)) scripts/generate_deb.sh: call fakeroot if available. +- ([#8409](https://github.com/EOSIO/eos/pull/8409)) Reflection validation script +- ([#8411](https://github.com/EOSIO/eos/pull/8411)) [develop] Github Actions for Community PRs +- ([#8413](https://github.com/EOSIO/eos/pull/8413)) Add better logging of exceptions in emit - develop +- ([#8424](https://github.com/EOSIO/eos/pull/8424)) fix discovery of openssl in tester cmake when OPENSSL_ROOT_DIR not set +- ([#8428](https://github.com/EOSIO/eos/pull/8428)) [develop] Fixing travis' source ~/.bash_profile problem +- ([#8433](https://github.com/EOSIO/eos/pull/8433)) [develop] Fix installation location of header file `eosio.version.hpp` +- ([#8437](https://github.com/EOSIO/eos/pull/8437)) abi serialization enhancements - develop +- ([#8444](https://github.com/EOSIO/eos/pull/8444)) resolve action return value hash & state history serialization discrepancy +- ([#8448](https://github.com/EOSIO/eos/pull/8448)) [Develop] Pipeline file for testing the build script +- ([#8453](https://github.com/EOSIO/eos/pull/8453)) [Develop] Added better sleep pre-execute for Anka commands + boost fix +- ([#8465](https://github.com/EOSIO/eos/pull/8465)) llvm 10 support for EOS VM OC +- ([#8466](https://github.com/EOSIO/eos/pull/8466)) [Develop] Switching to using the EOSIO fork of anka-buildkite-plugin for security reasons +- ([#8478](https://github.com/EOSIO/eos/pull/8478)) Update eos-vm +- ([#8484](https://github.com/EOSIO/eos/pull/8484)) [Develop] Fixes for Submodule Regression Checker Script +- ([#8486](https://github.com/EOSIO/eos/pull/8486)) [develop] Multiversion test migration +- ([#8489](https://github.com/EOSIO/eos/pull/8489)) Change link signature from state_history to state_history_plugin +- ([#8490](https://github.com/EOSIO/eos/pull/8490)) [develop] Preemptively create the wallet directory to prevent exception +- ([#8491](https://github.com/EOSIO/eos/pull/8491)) [develop] Docker name collision fix +- ([#8497](https://github.com/EOSIO/eos/pull/8497)) Drop late blocks - develop +- ([#8500](https://github.com/EOSIO/eos/pull/8500)) remove old WAVM Platform files and WAVM intrinsics +- ([#8501](https://github.com/EOSIO/eos/pull/8501)) [develop] Removed unnecessary sleep option from Anka plugin +- ([#8503](https://github.com/EOSIO/eos/pull/8503)) use sh instead of bash for cmake unittests magic +- ([#8505](https://github.com/EOSIO/eos/pull/8505)) Remove hash in link +- ([#8511](https://github.com/EOSIO/eos/pull/8511)) http_plugin shutdown - develop +- ([#8513](https://github.com/EOSIO/eos/pull/8513)) [develop] Don't trigger LRT a second time +- ([#8524](https://github.com/EOSIO/eos/pull/8524)) 2.0.1 security omnibus - develop +- ([#8527](https://github.com/EOSIO/eos/pull/8527)) Handle socket close before async callback - develop +- ([#8540](https://github.com/EOSIO/eos/pull/8540)) Added comparison operators for extended_symbol type +- ([#8548](https://github.com/EOSIO/eos/pull/8548)) Net plugin dispatch - develop +- ([#8550](https://github.com/EOSIO/eos/pull/8550)) Fix typo +- ([#8553](https://github.com/EOSIO/eos/pull/8553)) Net plugin unlinkable blocks - develop +- ([#8556](https://github.com/EOSIO/eos/pull/8556)) Drop late check - develop +- ([#8559](https://github.com/EOSIO/eos/pull/8559)) Read-only with drop-late-block - develop +- ([#8563](https://github.com/EOSIO/eos/pull/8563)) Net plugin post - develop +- ([#8565](https://github.com/EOSIO/eos/pull/8565)) Delayed production time - develop +- ([#8567](https://github.com/EOSIO/eos/pull/8567)) Timestamp watermark slot +- ([#8570](https://github.com/EOSIO/eos/pull/8570)) Eliminate use of boost deprecated query object. +- ([#8573](https://github.com/EOSIO/eos/pull/8573)) Anka / CICD 10.15.1 -> 10.15.3 +- ([#8579](https://github.com/EOSIO/eos/pull/8579)) CPU block effort - develop +- ([#8585](https://github.com/EOSIO/eos/pull/8585)) cpu effort last block - develop +- ([#8587](https://github.com/EOSIO/eos/pull/8587)) P2p read only - develop +- ([#8596](https://github.com/EOSIO/eos/pull/8596)) Consolidated Security Fixes for develop +- ([#8597](https://github.com/EOSIO/eos/pull/8597)) Producer plugin log - develop +- ([#8601](https://github.com/EOSIO/eos/pull/8601)) Improve create account description +- ([#8603](https://github.com/EOSIO/eos/pull/8603)) Skip sync from genesis and resume from state test on tagged builds +- ([#8609](https://github.com/EOSIO/eos/pull/8609)) Add a way to query nodeos reversible db size - added an api endpoint … +- ([#8613](https://github.com/EOSIO/eos/pull/8613)) [develop] Fixes for Actions. +- ([#8618](https://github.com/EOSIO/eos/pull/8618)) Init net_plugin member variables - develop +- ([#8623](https://github.com/EOSIO/eos/pull/8623)) abi 1.2: action_results +- ([#8635](https://github.com/EOSIO/eos/pull/8635)) bump script's macos version check to 10.14 +- ([#8637](https://github.com/EOSIO/eos/pull/8637)) remove brew's python@2 install +- ([#8646](https://github.com/EOSIO/eos/pull/8646)) Consolidated Security Fixes for develop. +- ([#8652](https://github.com/EOSIO/eos/pull/8652)) Fix format message. +- ([#8657](https://github.com/EOSIO/eos/pull/8657)) Fix wasm-runtime option parameters +- ([#8663](https://github.com/EOSIO/eos/pull/8663)) ship: add chain_id to get_status_result_v0 +- ([#8665](https://github.com/EOSIO/eos/pull/8665)) Fix other blocks.log callout +- ([#8669](https://github.com/EOSIO/eos/pull/8669)) Add troubleshooting item for PREACTIVATE_FEATURE protocol +- ([#8670](https://github.com/EOSIO/eos/pull/8670)) Using get raw abi in cleos +- ([#8671](https://github.com/EOSIO/eos/pull/8671)) Fix for cleos and keosd race condition +- ([#8674](https://github.com/EOSIO/eos/pull/8674)) [develop] Disable skip checkouts for EKS builder/tester fleet. +- ([#8676](https://github.com/EOSIO/eos/pull/8676)) unpack data when forming transaction, useful for … +- ([#8677](https://github.com/EOSIO/eos/pull/8677)) Allow Boost.Test to report the last checkpoint location when an excep… +- ([#8679](https://github.com/EOSIO/eos/pull/8679)) Exit transaction early when insufficient account cpu - develop +- ([#8681](https://github.com/EOSIO/eos/pull/8681)) Produce block immediately if exhausted - develop +- ([#8683](https://github.com/EOSIO/eos/pull/8683)) Produce time - develop +- ([#8687](https://github.com/EOSIO/eos/pull/8687)) Add Incoming-defer-ratio description +- ([#8688](https://github.com/EOSIO/eos/pull/8688)) Fixes #8600 clean up nodeos options section +- ([#8691](https://github.com/EOSIO/eos/pull/8691)) incoming-defer-ratio description - develop +- ([#8692](https://github.com/EOSIO/eos/pull/8692)) [develop] Community PR tweaks. +- ([#8699](https://github.com/EOSIO/eos/pull/8699)) [develop] Base images pipeline. +- ([#8704](https://github.com/EOSIO/eos/pull/8704)) add get_block_info +- ([#8706](https://github.com/EOSIO/eos/pull/8706)) Update the getting started link [merge 1] +- ([#8709](https://github.com/EOSIO/eos/pull/8709)) Relay block on accepted header - develop +- ([#8713](https://github.com/EOSIO/eos/pull/8713)) [develop] Actions rerun fixes. +- ([#8717](https://github.com/EOSIO/eos/pull/8717)) Fix mutliple version protocol test intermittent failure +- ([#8718](https://github.com/EOSIO/eos/pull/8718)) link cleos net status reference doc with the peer network protocol doc +- ([#8719](https://github.com/EOSIO/eos/pull/8719)) Add tests for multi_index iterator cache across notifies. +- ([#8720](https://github.com/EOSIO/eos/pull/8720)) Add unit test to verify that the description digests of protocol feat… +- ([#8728](https://github.com/EOSIO/eos/pull/8728)) remove the redundant html markup +- ([#8730](https://github.com/EOSIO/eos/pull/8730)) Add integrated Secure Enclave block signing for nodeos +- ([#8731](https://github.com/EOSIO/eos/pull/8731)) Get info priority - develop +- ([#8737](https://github.com/EOSIO/eos/pull/8737)) Fix/action results +- ([#8738](https://github.com/EOSIO/eos/pull/8738)) Add additional CPU/NET usage data to get_account results +- ([#8743](https://github.com/EOSIO/eos/pull/8743)) New options for api nodes - develop +- ([#8749](https://github.com/EOSIO/eos/pull/8749)) [CI/CD] -S to curl in generate-tag script so we can see why it's failing on EKS +- ([#8750](https://github.com/EOSIO/eos/pull/8750)) Move parts of state-history-plugin to libraries/state_history +- ([#8751](https://github.com/EOSIO/eos/pull/8751)) upgrade pinned builds to clang 10 & boost 1.72 +- ([#8755](https://github.com/EOSIO/eos/pull/8755)) add block producing explainer doc +- ([#8771](https://github.com/EOSIO/eos/pull/8771)) free unknown EOS VM OC codegen versions from the code cache +- ([#8779](https://github.com/EOSIO/eos/pull/8779)) disable EOS VM on non-x86 platforms +- ([#8780](https://github.com/EOSIO/eos/pull/8780)) link to librt when using posix timers +- ([#8788](https://github.com/EOSIO/eos/pull/8788)) dfuse Deep Mind changes +- ([#8801](https://github.com/EOSIO/eos/pull/8801)) Expire blacklisted scheduled transactions by LIB time - develop +- ([#8802](https://github.com/EOSIO/eos/pull/8802)) Trace API Plugin - develop +- ([#8812](https://github.com/EOSIO/eos/pull/8812)) disable temporarily snapshot creation +- ([#8818](https://github.com/EOSIO/eos/pull/8818)) Add test cases for changes of logging with minimize flag is true, +- ([#8820](https://github.com/EOSIO/eos/pull/8820)) yield_function for abi_serializer +- ([#8824](https://github.com/EOSIO/eos/pull/8824)) remove leading $ chars from shell codeblocks in README.md +- ([#8829](https://github.com/EOSIO/eos/pull/8829)) fix potential leak in OC's wrapped_fd move assignment op +- ([#8833](https://github.com/EOSIO/eos/pull/8833)) Add RPC Trace API plugin reference to nodeos +- ([#8834](https://github.com/EOSIO/eos/pull/8834)) trace_api_plugin yield timeout - develop +- ([#8838](https://github.com/EOSIO/eos/pull/8838)) set_action_return_value prohibited for context free actions +- ([#8842](https://github.com/EOSIO/eos/pull/8842)) Fix double titles in plugins +- ([#8846](https://github.com/EOSIO/eos/pull/8846)) skip context free actions during light validation +- ([#8847](https://github.com/EOSIO/eos/pull/8847)) add block replay test +- ([#8848](https://github.com/EOSIO/eos/pull/8848)) Skip checks +- ([#8851](https://github.com/EOSIO/eos/pull/8851)) add light validation sync test +- ([#8852](https://github.com/EOSIO/eos/pull/8852)) [develop] Trace API Compressed data log Support +- ([#8853](https://github.com/EOSIO/eos/pull/8853)) CFD: Initial support for pruned_block +- ([#8854](https://github.com/EOSIO/eos/pull/8854)) Improve too many bytes in flight error info - develop +- ([#8856](https://github.com/EOSIO/eos/pull/8856)) Use NET bill in transaction receipt during light validation mode +- ([#8864](https://github.com/EOSIO/eos/pull/8864)) wabt: don't search for python because we don't run tests +- ([#8865](https://github.com/EOSIO/eos/pull/8865)) Add possibility to run .cicd scripts from different environments +- ([#8868](https://github.com/EOSIO/eos/pull/8868)) Feature/new host function system +- ([#8874](https://github.com/EOSIO/eos/pull/8874)) Fix spurious HTTP related test failure [develop] (round 3) +- ([#8879](https://github.com/EOSIO/eos/pull/8879)) HTTP Plugin async APIs [develop] +- ([#8880](https://github.com/EOSIO/eos/pull/8880)) add pruned_block to signed_block conversion +- ([#8882](https://github.com/EOSIO/eos/pull/8882)) Correctly Sanitize git Branch and Tag Names +- ([#8886](https://github.com/EOSIO/eos/pull/8886)) use http async api support for Trace API get_block [develop] +- ([#8896](https://github.com/EOSIO/eos/pull/8896)) Increase get info priority to medium high - develop +- ([#8897](https://github.com/EOSIO/eos/pull/8897)) Sync from snapshot - develop +- ([#8898](https://github.com/EOSIO/eos/pull/8898)) Remove the assertion check for error code (400) in cleos +- ([#8905](https://github.com/EOSIO/eos/pull/8905)) Update eos-vm +- ([#8917](https://github.com/EOSIO/eos/pull/8917)) Updates to manual build instructions +- ([#8922](https://github.com/EOSIO/eos/pull/8922)) remove left over support patch for previous clang 8 pinned compiler +- ([#8924](https://github.com/EOSIO/eos/pull/8924)) Add unwrapped chainlib +- ([#8925](https://github.com/EOSIO/eos/pull/8925)) remove llvm@7 from macos build as it isn't used at the moment +- ([#8927](https://github.com/EOSIO/eos/pull/8927)) Fix SHIP block delay - develop +- ([#8928](https://github.com/EOSIO/eos/pull/8928)) replace boost::bind with std::bind, fixing boost 1.73beta builds +- ([#8929](https://github.com/EOSIO/eos/pull/8929)) Chainlib support for replacing keys +- ([#8930](https://github.com/EOSIO/eos/pull/8930)) fix boost URL in mojave cicd script +- ([#8931](https://github.com/EOSIO/eos/pull/8931)) Fix unpack data for signing transaction +- ([#8932](https://github.com/EOSIO/eos/pull/8932)) Rename action_id type for GCC - develop +- ([#8937](https://github.com/EOSIO/eos/pull/8937)) Fix broken Docker build of C7 pinned image. +- ([#8958](https://github.com/EOSIO/eos/pull/8958)) Replace bc with shell arithmetic - develop +- ([#8959](https://github.com/EOSIO/eos/pull/8959)) Make /bin/df ignore $BLOCKSIZE - develop +- ([#8960](https://github.com/EOSIO/eos/pull/8960)) Upgrade CLI11 to 1.9.0 - develop +- ([#8961](https://github.com/EOSIO/eos/pull/8961)) Support Running ALL Tests in One Build +- ([#8964](https://github.com/EOSIO/eos/pull/8964)) unit-test for replace keys +- ([#8966](https://github.com/EOSIO/eos/pull/8966)) [develop] Bump Catalina version. +- ([#8967](https://github.com/EOSIO/eos/pull/8967)) tests/get_table_tests.cpp: incorrect use of CORE_SYM_STR - develop +- ([#8979](https://github.com/EOSIO/eos/pull/8979)) Add nodeos RPC API index, improve nodeos implementation doc, fix link +- ([#8991](https://github.com/EOSIO/eos/pull/8991)) Avoid legacy for set_action_return_value intrinsic +- ([#8994](https://github.com/EOSIO/eos/pull/8994)) Update example logging.json - develop +- ([#8998](https://github.com/EOSIO/eos/pull/8998)) Better error handling for push/send_transaction - develop +- ([#8999](https://github.com/EOSIO/eos/pull/8999)) Fixed failing nodeos_run_test when core symbol is not SYS - develop +- ([#9000](https://github.com/EOSIO/eos/pull/9000)) Improved reporting in nodeos_forked_chain_lr_test +- ([#9001](https://github.com/EOSIO/eos/pull/9001)) Support Triggering a Build that Runs ALL Tests in One Build +- ([#9011](https://github.com/EOSIO/eos/pull/9011)) Revert "Upgrade CLI11 to 1.9.0 - develop" +- ([#9012](https://github.com/EOSIO/eos/pull/9012)) Bugfix for uninitialized variable in cleos - develop +- ([#9015](https://github.com/EOSIO/eos/pull/9015)) Bump version to 2.1.0-alpha1 +- ([#9016](https://github.com/EOSIO/eos/pull/9016)) Bring back CLI11 1.9.0 - develop +- ([#9018](https://github.com/EOSIO/eos/pull/9018)) rodeos and eosio-tester +- ([#9019](https://github.com/EOSIO/eos/pull/9019)) refactor block log +- ([#9020](https://github.com/EOSIO/eos/pull/9020)) add help text to wasm-runtime - develop +- ([#9021](https://github.com/EOSIO/eos/pull/9021)) Add authority structure to cleos system newaccount +- ([#9025](https://github.com/EOSIO/eos/pull/9025)) Fix keosd auto-launching after CLI11 upgrade - develop +- ([#9029](https://github.com/EOSIO/eos/pull/9029)) Rodeos with Streaming Plugin +- ([#9033](https://github.com/EOSIO/eos/pull/9033)) Adding message body check (400) for http calls +- ([#9034](https://github.com/EOSIO/eos/pull/9034)) sync fc up to master bringing 3 PRs in +- ([#9039](https://github.com/EOSIO/eos/pull/9039)) For develop - Updated the priority of the APIs in producer_api_plugin and net_api_plugin to MEDIUM_HIGH +- ([#9041](https://github.com/EOSIO/eos/pull/9041)) move minimum boost from 1.67->1.70; gcc 7->8 +- ([#9043](https://github.com/EOSIO/eos/pull/9043)) Remove copy of result - develop +- ([#9044](https://github.com/EOSIO/eos/pull/9044)) Replace submodules +- ([#9046](https://github.com/EOSIO/eos/pull/9046)) Remove outcome +- ([#9047](https://github.com/EOSIO/eos/pull/9047)) [develop]Add more info in trace-api-plugin +- ([#9048](https://github.com/EOSIO/eos/pull/9048)) add rapidjson license to install - develop +- ([#9050](https://github.com/EOSIO/eos/pull/9050)) Add cleos --compression option for transactions +- ([#9051](https://github.com/EOSIO/eos/pull/9051)) removed unused cmake modules from fc +- ([#9053](https://github.com/EOSIO/eos/pull/9053)) Print stderr if keosd_auto_launch_test.py fails - develop +- ([#9054](https://github.com/EOSIO/eos/pull/9054)) add options for not using GMP and for static linking GMP +- ([#9057](https://github.com/EOSIO/eos/pull/9057)) Fix timedelta and strftime usage - develop +- ([#9059](https://github.com/EOSIO/eos/pull/9059)) Fix uninitialized struct members used as CLI flags - develop +- ([#9061](https://github.com/EOSIO/eos/pull/9061)) Merge prune-cfd-stage-1 branch +- ([#9066](https://github.com/EOSIO/eos/pull/9066)) separate out signature provider from producer plugin +- ([#9068](https://github.com/EOSIO/eos/pull/9068)) add cleos validate signatures +- ([#9069](https://github.com/EOSIO/eos/pull/9069)) Use `signed_block_v0` binary format for SHiP +- ([#9070](https://github.com/EOSIO/eos/pull/9070)) fix two range-loop-construct warnings from clang10 +- ([#9072](https://github.com/EOSIO/eos/pull/9072)) CFD pruning integration test +- ([#9074](https://github.com/EOSIO/eos/pull/9074)) Add change type to pull request template +- ([#9077](https://github.com/EOSIO/eos/pull/9077)) Update date in LICENSE +- ([#9079](https://github.com/EOSIO/eos/pull/9079)) Fix setting of keosd-provider-timeout +- ([#9080](https://github.com/EOSIO/eos/pull/9080)) Add support for specifying a logging.json to keosd - develop +- ([#9081](https://github.com/EOSIO/eos/pull/9081)) ship v0 fix +- ([#9085](https://github.com/EOSIO/eos/pull/9085)) trim-blocklog improvement (removing bad blocks and making blocks.log … +- ([#9086](https://github.com/EOSIO/eos/pull/9086)) Add back transaction de-duplication check in net_plugin +- ([#9088](https://github.com/EOSIO/eos/pull/9088)) make ship WA key serialization match expected serialization +- ([#9092](https://github.com/EOSIO/eos/pull/9092)) Fix narrowing conversion error in `fc/src/log/console_appender.cpp` +- ([#9094](https://github.com/EOSIO/eos/pull/9094)) fix gcc10 build due to libyubihsm problem +- ([#9104](https://github.com/EOSIO/eos/pull/9104)) Ship v1 +- ([#9108](https://github.com/EOSIO/eos/pull/9108)) [develop] Bump MacOS version and timeouts. +- ([#9111](https://github.com/EOSIO/eos/pull/9111)) Update algorithm for determining number of parallel jobs - develop +- ([#9114](https://github.com/EOSIO/eos/pull/9114)) [develop] Epe 37 fix test contracts build +- ([#9117](https://github.com/EOSIO/eos/pull/9117)) Exit on rodeos filter wasm error +- ([#9119](https://github.com/EOSIO/eos/pull/9119)) fixes amqp heartbeat idle connection +- ([#9123](https://github.com/EOSIO/eos/pull/9123)) Update the authority example JSON +- ([#9125](https://github.com/EOSIO/eos/pull/9125)) Add unity build support for some targets +- ([#9126](https://github.com/EOSIO/eos/pull/9126)) Fix onblock handling in trace_api_plugin - develop +- ([#9132](https://github.com/EOSIO/eos/pull/9132)) Rodeos streamer exchanges +- ([#9133](https://github.com/EOSIO/eos/pull/9133)) Restore abi_serializer backward compatibility - develop +- ([#9134](https://github.com/EOSIO/eos/pull/9134)) Test framework archiving +- ([#9137](https://github.com/EOSIO/eos/pull/9137)) Fix api notification of applied trx +- ([#9143](https://github.com/EOSIO/eos/pull/9143)) Prune data integration test fix +- ([#9147](https://github.com/EOSIO/eos/pull/9147)) two comment fixes to transaction.hpp +- ([#9149](https://github.com/EOSIO/eos/pull/9149)) Fix for empty ("") appbase config default value +- ([#9160](https://github.com/EOSIO/eos/pull/9160)) fix build when build path has spaces +- ([#9164](https://github.com/EOSIO/eos/pull/9164)) Fix for connection cycle not being in sync with test startup. +- ([#9165](https://github.com/EOSIO/eos/pull/9165)) fix helper for CLANG 10 detection +- ([#9167](https://github.com/EOSIO/eos/pull/9167)) stop rocksdb's CMakeLists from force overriding CMAKE_INSTALL_PREFIX +- ([#9169](https://github.com/EOSIO/eos/pull/9169)) Fix onblock trace tracking - develop +- ([#9175](https://github.com/EOSIO/eos/pull/9175)) Ship delay error fix +- ([#9179](https://github.com/EOSIO/eos/pull/9179)) Add a sign intrinsic to the tester. +- ([#9180](https://github.com/EOSIO/eos/pull/9180)) eosio.contracts unit tests fail to compile with develop branch due to controller change +- ([#9182](https://github.com/EOSIO/eos/pull/9182)) Bump to alpha2 +- ([#9184](https://github.com/EOSIO/eos/pull/9184)) Add support for block log splitting +- ([#9186](https://github.com/EOSIO/eos/pull/9186)) struct name fix check #8971 +- ([#9187](https://github.com/EOSIO/eos/pull/9187)) Fixed relaunch calls that still passed in nodeId. +- ([#9194](https://github.com/EOSIO/eos/pull/9194)) Add trace plugin API test +- ([#9196](https://github.com/EOSIO/eos/pull/9196)) Resource monitor plugin -- develop branch +- ([#9198](https://github.com/EOSIO/eos/pull/9198)) Reenable OC and update it to the new intrinsic wrappers. +- ([#9199](https://github.com/EOSIO/eos/pull/9199)) [develop] Anka/Catalina version bump +- ([#9204](https://github.com/EOSIO/eos/pull/9204)) Support unity build for unittests +- ([#9207](https://github.com/EOSIO/eos/pull/9207)) call boost program option notifiers before plugin initialize +- ([#9209](https://github.com/EOSIO/eos/pull/9209)) add empty content http request handling +- ([#9210](https://github.com/EOSIO/eos/pull/9210)) Fix eosio-blocklog trim front +- ([#9211](https://github.com/EOSIO/eos/pull/9211)) Loosen production round requirement +- ([#9212](https://github.com/EOSIO/eos/pull/9212)) Apply 400 check to db_size +- ([#9213](https://github.com/EOSIO/eos/pull/9213)) Replace fc::optional with std::optional +- ([#9217](https://github.com/EOSIO/eos/pull/9217)) Improve parsing of RabbitMQ-related command line arguments in rodeos - develop +- ([#9218](https://github.com/EOSIO/eos/pull/9218)) EPE-145: unapplied_transaction_queue incorrectly caches incoming_count +- ([#9221](https://github.com/EOSIO/eos/pull/9221)) Fix unity build for unittests +- ([#9222](https://github.com/EOSIO/eos/pull/9222)) Fix log of pending block producer - develop +- ([#9226](https://github.com/EOSIO/eos/pull/9226)) call q.begin and q.end, instead of q.unapplied_begin and q.unapplied_end, in unit tests +- ([#9231](https://github.com/EOSIO/eos/pull/9231)) Comment clean up +- ([#9233](https://github.com/EOSIO/eos/pull/9233)) Changed code to ensure --http-max-response-time-ms is always passed in the extraNodeosArgs +- ([#9235](https://github.com/EOSIO/eos/pull/9235)) Migrate fc::static_variant to std::variant +- ([#9239](https://github.com/EOSIO/eos/pull/9239)) split transaction logging +- ([#9244](https://github.com/EOSIO/eos/pull/9244)) relaxing the on_notify constraint to * +- ([#9245](https://github.com/EOSIO/eos/pull/9245)) added a new option fix-irreversible-blocks +- ([#9248](https://github.com/EOSIO/eos/pull/9248)) add test case to restart chain without blocks.log +- ([#9253](https://github.com/EOSIO/eos/pull/9253)) Additional ShIP unit tests +- ([#9254](https://github.com/EOSIO/eos/pull/9254)) const correctness fix +- ([#9257](https://github.com/EOSIO/eos/pull/9257)) add new loggers to logging.json +- ([#9263](https://github.com/EOSIO/eos/pull/9263)) Remove Concurrency Groups for Scheduled Builds +- ([#9277](https://github.com/EOSIO/eos/pull/9277)) Support state history log splitting +- ([#9277](https://github.com/EOSIO/eos/pull/9277)) Support state history log splitting +- ([#9281](https://github.com/EOSIO/eos/pull/9281)) Refactor to use std::unique_ptr instead of naked pointers +- ([#9289](https://github.com/EOSIO/eos/pull/9289)) add covert_to_type for name +- ([#9308](https://github.com/EOSIO/eos/pull/9308)) Track Source Files Excluded from Code Coverage Reports +- ([#9310](https://github.com/EOSIO/eos/pull/9310)) Add action result to abi serializer +- ([#9317](https://github.com/EOSIO/eos/pull/9317)) fix UB with rvalue reference +- ([#9328](https://github.com/EOSIO/eos/pull/9328)) Fix core dump on logging when no this_block set +- ([#9332](https://github.com/EOSIO/eos/pull/9332)) updated scripts to support Ubuntu 20.04 +- ([#9333](https://github.com/EOSIO/eos/pull/9333)) Use fc::variant() instead of 0 to be clearer that value is not available +- ([#9337](https://github.com/EOSIO/eos/pull/9337)) Make shutdown() private as it should only be called from quit() +- ([#9342](https://github.com/EOSIO/eos/pull/9342)) Fix typo in pull request template +- ([#9347](https://github.com/EOSIO/eos/pull/9347)) Update abieos submodule to point to eosio branch +- ([#9351](https://github.com/EOSIO/eos/pull/9351)) Nonprivileged inline action subjective limit - develop +- ([#9353](https://github.com/EOSIO/eos/pull/9353)) Update CLI11 to v1.9.1 +- ([#9354](https://github.com/EOSIO/eos/pull/9354)) Add overload to serializer for action_traces in order to deserialize action return values +- ([#9362](https://github.com/EOSIO/eos/pull/9362)) Consolidated security fixes +- ([#9364](https://github.com/EOSIO/eos/pull/9364)) Add Ubuntu 20.04 cicd dockerfiles/buildscripts-develop +- ([#9368](https://github.com/EOSIO/eos/pull/9368)) Remove unnecessary strlen +- ([#9369](https://github.com/EOSIO/eos/pull/9369)) set medium priority for process signed block - develop +- ([#9371](https://github.com/EOSIO/eos/pull/9371)) Reenable snapshot tests +- ([#9375](https://github.com/EOSIO/eos/pull/9375)) cleos to display pushed actions' return values +- ([#9381](https://github.com/EOSIO/eos/pull/9381)) add std::list<> support to fc pack/unpack (develop) +- ([#9383](https://github.com/EOSIO/eos/pull/9383)) Read transaction consensus fix +- ([#9384](https://github.com/EOSIO/eos/pull/9384)) develop version of "Account Query DB : maintain get_(key|controlled)_accounts" +- ([#9385](https://github.com/EOSIO/eos/pull/9385)) Remove deprecated functions in abi_serializer for EPE112 +- ([#9389](https://github.com/EOSIO/eos/pull/9389)) Remove fc::uint128_t typedef +- ([#9390](https://github.com/EOSIO/eos/pull/9390)) test contracts fix +- ([#9392](https://github.com/EOSIO/eos/pull/9392)) EPE-306 fix +- ([#9393](https://github.com/EOSIO/eos/pull/9393)) fix macos build script on Big Sur +- ([#9395](https://github.com/EOSIO/eos/pull/9395)) Enable the correct lrt for snapshot generation testing +- ([#9398](https://github.com/EOSIO/eos/pull/9398)) [develop] Fix docker tags when building forked PRs +- ([#9401](https://github.com/EOSIO/eos/pull/9401)) set max_irreversible_block_age to -1 +- ([#9403](https://github.com/EOSIO/eos/pull/9403)) Increse max_transaction_cpu_usage to 90k +- ([#9405](https://github.com/EOSIO/eos/pull/9405)) added unit tests +- ([#9410](https://github.com/EOSIO/eos/pull/9410)) Cleos http response handler develop +- ([#9411](https://github.com/EOSIO/eos/pull/9411)) fix the bug that the flight bytes are cacculated incorrect +- ([#9416](https://github.com/EOSIO/eos/pull/9416)) fix template instantiation for host function +- ([#9420](https://github.com/EOSIO/eos/pull/9420)) Fix variant type blob unpack bug +- ([#9427](https://github.com/EOSIO/eos/pull/9427)) Fix static initialization problem +- ([#9429](https://github.com/EOSIO/eos/pull/9429)) Abi kv nodeos +- ([#9431](https://github.com/EOSIO/eos/pull/9431)) Restrict the maximum number of open HTTP RPC requests +- ([#9432](https://github.com/EOSIO/eos/pull/9432)) resolve inconsistent visibility warnings on mac +- ([#9433](https://github.com/EOSIO/eos/pull/9433)) fix build problem for git absence +- ([#9434](https://github.com/EOSIO/eos/pull/9434)) Fix unnecessary object copying +- ([#9435](https://github.com/EOSIO/eos/pull/9435)) update abieos submodule +- ([#9440](https://github.com/EOSIO/eos/pull/9440)) Fix app() shutdown - develop +- ([#9444](https://github.com/EOSIO/eos/pull/9444)) remove unity build +- ([#9445](https://github.com/EOSIO/eos/pull/9445)) move is_string_valid_name to cpp file +- ([#9447](https://github.com/EOSIO/eos/pull/9447)) Replace N macro with operator ""_n - develop +- ([#9448](https://github.com/EOSIO/eos/pull/9448)) Fix develop build +- ([#9449](https://github.com/EOSIO/eos/pull/9449)) Support for storing kv and db intrinsics in Chainbase or RocksDB. +- ([#9451](https://github.com/EOSIO/eos/pull/9451)) new chain_config param: action return value limit +- ([#9453](https://github.com/EOSIO/eos/pull/9453)) Reverting some libs +- ([#9460](https://github.com/EOSIO/eos/pull/9460)) rpc kv access implement get_kv_table_rows +- ([#9461](https://github.com/EOSIO/eos/pull/9461)) fix slipped submod +- ([#9468](https://github.com/EOSIO/eos/pull/9468)) added try catch +- ([#9475](https://github.com/EOSIO/eos/pull/9475)) Add script support for CentOS 8 (redo of #9361) +- ([#9477](https://github.com/EOSIO/eos/pull/9477)) Add first class support for converting ABIs themselves to/from json/bin/hex +- ([#9486](https://github.com/EOSIO/eos/pull/9486)) Fix build - N macro was removed +- ([#9494](https://github.com/EOSIO/eos/pull/9494)) add an integration of nodeos for crash when the nodes are killed +- ([#9499](https://github.com/EOSIO/eos/pull/9499)) add accessor for controller's trusted producer list +- ([#9512](https://github.com/EOSIO/eos/pull/9512)) Keep http_plugin_impl alive while connection objects are alive +- ([#9514](https://github.com/EOSIO/eos/pull/9514)) Fix for broken Centos 8 build-scripts build +- ([#9517](https://github.com/EOSIO/eos/pull/9517)) Update abieos with change of to_json may_not_exist fields +- ([#9520](https://github.com/EOSIO/eos/pull/9520)) Add installation pkg to centos 7 build deps and centos script +- ([#9524](https://github.com/EOSIO/eos/pull/9524)) fix centOS 8 test failures +- ([#9533](https://github.com/EOSIO/eos/pull/9533)) Failure with building on Centos 7.x +- ([#9536](https://github.com/EOSIO/eos/pull/9536)) kv support cleos +- ([#9546](https://github.com/EOSIO/eos/pull/9546)) add combined_db kv_context +- ([#9547](https://github.com/EOSIO/eos/pull/9547)) Trace API plugin - Add support for action return values +- ([#9553](https://github.com/EOSIO/eos/pull/9553)) fix secondary index in get_kv_table_rows +- ([#9566](https://github.com/EOSIO/eos/pull/9566)) Removing unused variable functionDefIndex +- ([#9577](https://github.com/EOSIO/eos/pull/9577)) use huge pages via mmap() instead of hugetlbfs +- ([#9582](https://github.com/EOSIO/eos/pull/9582)) Fix stdout console logging +- ([#9593](https://github.com/EOSIO/eos/pull/9593)) Speculative validation optimizations +- ([#9595](https://github.com/EOSIO/eos/pull/9595)) fixed cleos get_kv_table_rows bugs +- ([#9596](https://github.com/EOSIO/eos/pull/9596)) restore dropped commit from fc resubmod: GMP options +- ([#9600](https://github.com/EOSIO/eos/pull/9600)) Session optimizations +- ([#9605](https://github.com/EOSIO/eos/pull/9605)) fix get_table_rows_by_seckey conversion +- ([#9607](https://github.com/EOSIO/eos/pull/9607)) Fix test_pending_schedule_snapshot by using blocks.log approach to ma… +- ([#9611](https://github.com/EOSIO/eos/pull/9611)) RocksDB temporary fix +- ([#9614](https://github.com/EOSIO/eos/pull/9614)) updated appbase to fix print-default-config for wasm-runtime +- ([#9615](https://github.com/EOSIO/eos/pull/9615)) only use '#pragma clang diagnostic' when compiling with clang +- ([#9622](https://github.com/EOSIO/eos/pull/9622)) Making create_snapshot output more informative by adding more fields +- ([#9623](https://github.com/EOSIO/eos/pull/9623)) Migrate CI from Docker Hub to Amazon ECR +- ([#9625](https://github.com/EOSIO/eos/pull/9625)) Fixing typos on injected params +- ([#9628](https://github.com/EOSIO/eos/pull/9628)) Misc tests +- ([#9631](https://github.com/EOSIO/eos/pull/9631)) Zipkin - develop +- ([#9632](https://github.com/EOSIO/eos/pull/9632)) Fixes for DB intrinsic replay logic +- ([#9633](https://github.com/EOSIO/eos/pull/9633)) Allow HTTP-RPC with empty response +- ([#9635](https://github.com/EOSIO/eos/pull/9635)) Update SHiP to work with RocksDB +- ([#9646](https://github.com/EOSIO/eos/pull/9646)) fix get_kv_table_rows secondary index search +- ([#9648](https://github.com/EOSIO/eos/pull/9648)) updated unit test kv_addr_book +- ([#9656](https://github.com/EOSIO/eos/pull/9656)) CI: Fix Serial Test Bug + Simplification + UX +- ([#9659](https://github.com/EOSIO/eos/pull/9659)) fix sprintf overrun +- ([#9660](https://github.com/EOSIO/eos/pull/9660)) resolve some warnings w.r.t. copying from consts +- ([#9662](https://github.com/EOSIO/eos/pull/9662)) Add "Testing Changes" Section to Pull Request Template +- ([#9667](https://github.com/EOSIO/eos/pull/9667)) Add "Ubuntu 20.04 Package Builder" step to pipeline.yml +- ([#9669](https://github.com/EOSIO/eos/pull/9669)) ship delta changes for issue 9255 +- ([#9670](https://github.com/EOSIO/eos/pull/9670)) disable building rodeos and eosio.tester +- ([#9673](https://github.com/EOSIO/eos/pull/9673)) restore boost 1.67 as the minimum boost version required +- ([#9674](https://github.com/EOSIO/eos/pull/9674)) Move chainbase calls out of try-CATCH_AND_EXIT_DB_FAILURE block +- ([#9680](https://github.com/EOSIO/eos/pull/9680)) add fc change of add reason to copy +- ([#9681](https://github.com/EOSIO/eos/pull/9681)) warning fix +- ([#9685](https://github.com/EOSIO/eos/pull/9685)) Rocksdb rpc support +- ([#9686](https://github.com/EOSIO/eos/pull/9686)) Pop back a delta with empty rows #9386 +- ([#9692](https://github.com/EOSIO/eos/pull/9692)) RocksDB - Renaming / creation of some parameters and change of default value for create_if_missing +- ([#9694](https://github.com/EOSIO/eos/pull/9694)) net_plugin monitor heartbeat of peers +- ([#9696](https://github.com/EOSIO/eos/pull/9696)) add fc support for boost 74 file copy +- ([#9707](https://github.com/EOSIO/eos/pull/9707)) Updated unit tests for new SHiP delta present field semantics +- ([#9712](https://github.com/EOSIO/eos/pull/9712)) Snapshot memory exhaustion +- ([#9713](https://github.com/EOSIO/eos/pull/9713)) Updating abieos to the latest abieos on eosio branch +- ([#9716](https://github.com/EOSIO/eos/pull/9716)) eosio-bios and eosio-boot contracts support for KV inside eosio + +## Documentation +- ([#7758](https://github.com/EOSIO/eos/pull/7758)) [wip] Add cleos, keosd doc outline and content +- ([#7963](https://github.com/EOSIO/eos/pull/7963)) Update README.md +- ([#8369](https://github.com/EOSIO/eos/pull/8369)) Update EOSIO documentation (develop) +- ([#8436](https://github.com/EOSIO/eos/pull/8436)) [develop] hotfix documentation links in README.md +- ([#8494](https://github.com/EOSIO/eos/pull/8494)) chain_api_plugin swagger file - develop +- ([#8576](https://github.com/EOSIO/eos/pull/8576)) [develop] Documentation patch 1 update +- ([#8666](https://github.com/EOSIO/eos/pull/8666)) Fix broken link in producer plugin docs +- ([#8809](https://github.com/EOSIO/eos/pull/8809)) Add initial Trace API plugin docs to nodeos +- ([#8827](https://github.com/EOSIO/eos/pull/8827)) db_size_api_plugin swagger file +- ([#8828](https://github.com/EOSIO/eos/pull/8828)) net_api_plugin swagger file +- ([#8830](https://github.com/EOSIO/eos/pull/8830)) producer_api_plugin swagger file +- ([#8831](https://github.com/EOSIO/eos/pull/8831)) test_control_api_plugin swagger +- ([#8832](https://github.com/EOSIO/eos/pull/8832)) swagger configuration for docs +- ([#8844](https://github.com/EOSIO/eos/pull/8844)) Trace API documentation update +- ([#8921](https://github.com/EOSIO/eos/pull/8921)) [docs] trace api reference api correction +- ([#9091](https://github.com/EOSIO/eos/pull/9091)) [docs] Add cleos validate signatures command reference +- ([#9150](https://github.com/EOSIO/eos/pull/9150)) Fix inaccurate nodeos reference in wallet_api_plugin [docs] +- ([#9151](https://github.com/EOSIO/eos/pull/9151)) Add default contract name clarifier in how to deploy smart contract [docs] +- ([#9152](https://github.com/EOSIO/eos/pull/9152)) Add trace_api logger [docs] +- ([#9153](https://github.com/EOSIO/eos/pull/9153)) Simplify create_snapshot POST request [docs] +- ([#9154](https://github.com/EOSIO/eos/pull/9154)) Replace inaccurate wording in how to replay from snapshot [docs] +- ([#9155](https://github.com/EOSIO/eos/pull/9155)) Fix Trace API reference request/response inaccuracies [docs] +- ([#9156](https://github.com/EOSIO/eos/pull/9156)) Add missing reference to RPC API index [docs] +- ([#9157](https://github.com/EOSIO/eos/pull/9157)) Fix title case issue in keosd how-to [docs] +- ([#9158](https://github.com/EOSIO/eos/pull/9158)) Add conditional step in state history plugin how-to [docs] +- ([#9208](https://github.com/EOSIO/eos/pull/9208)) add separate logging for state history plugin +- ([#9270](https://github.com/EOSIO/eos/pull/9270)) New threshold for non privileged inline actions +- ([#9279](https://github.com/EOSIO/eos/pull/9279)) [docs] Correct Producer API title in RPC reference +- ([#9291](https://github.com/EOSIO/eos/pull/9291)) [docs] Fix character formatting in nodeos CLI option +- ([#9320](https://github.com/EOSIO/eos/pull/9320)) [docs] Remove redundant nodeos replay example +- ([#9321](https://github.com/EOSIO/eos/pull/9321)) [docs] Remove unneeded options for nodeos replays +- ([#9339](https://github.com/EOSIO/eos/pull/9339)) [docs] Add chain plugin options that support SHiP logging +- ([#9374](https://github.com/EOSIO/eos/pull/9374)) [docs] Fix broken link in Wallet API plugin +- ([#9400](https://github.com/EOSIO/eos/pull/9400)) [docs] add return value from actions cleos output explanation and samples +- ([#9465](https://github.com/EOSIO/eos/pull/9465)) [docs] Create nodeos concepts folder and rearrange folders +- ([#9466](https://github.com/EOSIO/eos/pull/9466)) Fix missing whitespace in yaml chain_api_plugin swagger +- ([#9470](https://github.com/EOSIO/eos/pull/9470)) [docs] Fix documentation how-to for delegating cpu with cleos +- ([#9471](https://github.com/EOSIO/eos/pull/9471)) [docs] Fix documentation how-to for delegating net with cleos +- ([#9504](https://github.com/EOSIO/eos/pull/9504)) [docs] Add prune CFD explainers, how-tos, utilities +- ([#9506](https://github.com/EOSIO/eos/pull/9506)) [docs] Add slices, trace log, clog format explainers to Trace API plugin +- ([#9508](https://github.com/EOSIO/eos/pull/9508)) [docs] Add WASM interface C++ reference documentation +- ([#9509](https://github.com/EOSIO/eos/pull/9509)) [docs] Update supported OS platforms for EOSIO 2.1 +- ([#9557](https://github.com/EOSIO/eos/pull/9557)) [docs] Add get_block_info RPC reference and use 3.0 schemata links +- ([#9561](https://github.com/EOSIO/eos/pull/9561)) Adding state store config docs +- ([#9565](https://github.com/EOSIO/eos/pull/9565)) [docs] Add trace_api_util reference to eosio utilities docs +- ([#9581](https://github.com/EOSIO/eos/pull/9581)) Make bios-boot-tutorial.py not rely on prior version of system contracts +- ([#9583](https://github.com/EOSIO/eos/pull/9583)) [docs] Add cleos get kv_table reference documentation +- ([#9590](https://github.com/EOSIO/eos/pull/9590)) [docs] Various additions/fixes to cleos reference +- ([#9601](https://github.com/EOSIO/eos/pull/9601)) [docs] Fix broken anchor link on MacOS build from source +- ([#9606](https://github.com/EOSIO/eos/pull/9606)) last_irreversible_block_time added to get_info API +- ([#9618](https://github.com/EOSIO/eos/pull/9618)) [docs] Update cleos get kv_table reference +- ([#9630](https://github.com/EOSIO/eos/pull/9630)) [docs] Update get_table_* reference in Chain API +- ([#9687](https://github.com/EOSIO/eos/pull/9687)) [docs] adding third party logging and tracing integration documentation for + +## Thanks! +Special thanks to the community contributors that submitted patches for this release: +- @MrToph +- @conr2d +- @javierjmc diff --git a/docs/30_release-notes/index.md b/docs/30_release-notes/index.md index 067b03c515c..ab9e8592db3 100644 --- a/docs/30_release-notes/index.md +++ b/docs/30_release-notes/index.md @@ -1,20 +1,65 @@ --- -content_title: EOSIO v2.0.11 Release Notes +content_title: EOSIO v2.1.0-rc3 Release Notes --- -This release contains security and miscellaneous fixes. +This is a ***RELEASE CANDIDATE*** for version 2.1.0. + +This release contains security, stability, and miscellaneous fixes. ## Security bug fixes -### Consolidated Security Fixes for v2.0.11 ([#10147](https://github.com/EOSIO/eos/pull/10147)) -- Fix issue with account query db that could result in incorrect data or hung processes +### Consolidated Security Fixes for v2.1.0-rc3 ([#9869](https://github.com/EOSIO/eos/pull/9869)) +- Fixes to packed_transaction cache +- Transaction account fail limit refactor Note: These security fixes are relevant to all nodes on EOSIO blockchain networks. +## Stability bug fixes +- ([#9864](https://github.com/EOSIO/eos/pull/9864)) fix incorrect transaction_extensions declaration +- ([#9880](https://github.com/EOSIO/eos/pull/9880)) Fix ship big vector serialization +- ([#9896](https://github.com/EOSIO/eos/pull/9896)) Fix state_history zlib_unpack bug +- ([#9909](https://github.com/EOSIO/eos/pull/9909)) Fix state_history::length_writer +- ([#9986](https://github.com/EOSIO/eos/pull/9986)) EPE-389 fix net_plugin stall during head_catchup - merge release/2.1.x +- ([#9988](https://github.com/EOSIO/eos/pull/9988)) refactor kv get rows 2.1.x +- ([#9989](https://github.com/EOSIO/eos/pull/9989)) Explicit ABI conversion of signed_transaction - merge 2.1.x +- ([#10027](https://github.com/EOSIO/eos/pull/10027)) EPE-165: Improve logic for unlinkable blocks while sync'ing +- ([#10028](https://github.com/EOSIO/eos/pull/10028)) use p2p address for duplicate connection resolution + ## Other changes -- ([#10063](https://github.com/EOSIO/eos/pull/10063)) [release 2.0.x] Fix docker steps on tagged builds. -- ([#10135](https://github.com/EOSIO/eos/pull/10135)) Wlb/adding nodeos param tests 2.0.x -- ([#10133](https://github.com/EOSIO/eos/pull/10133)) fix compiling with boost 1.76; add include in chainbase - 2.0 +- ([#9858](https://github.com/EOSIO/eos/pull/9858)) Fix problem when using ubuntu libpqxx package +- ([#9863](https://github.com/EOSIO/eos/pull/9863)) chain_plugin db intrinsic table RPC calls incorrectly handling --lower and --upper in certain scenarios +- ([#9882](https://github.com/EOSIO/eos/pull/9882)) merge back fix build problem on cmake3.10 +- ([#9884](https://github.com/EOSIO/eos/pull/9884)) Fix problem with libpqxx 7.3.0 upgrade +- ([#9893](https://github.com/EOSIO/eos/pull/9893)) EOS VM OC: Support LLVM 11 - 2.1 +- ([#9900](https://github.com/EOSIO/eos/pull/9900)) Create Docker image with the eos binary and push to Dockerhub +- ([#9906](https://github.com/EOSIO/eos/pull/9906)) Add log path for unsupported log version exception +- ([#9930](https://github.com/EOSIO/eos/pull/9930)) Fix intermittent forked chain test failure +- ([#9931](https://github.com/EOSIO/eos/pull/9931)) trace history log messages should print nicely in syslog +- ([#9942](https://github.com/EOSIO/eos/pull/9942)) Fix "cleos net peers" command error +- ([#9943](https://github.com/EOSIO/eos/pull/9943)) Create eosio-debug-build Pipeline +- ([#9953](https://github.com/EOSIO/eos/pull/9953)) EPE-482 Fixed warning due to unreferenced label +- ([#9956](https://github.com/EOSIO/eos/pull/9956)) PowerTools is now powertools in CentOS 8.3 - 2.1 +- ([#9958](https://github.com/EOSIO/eos/pull/9958)) merge back PR 9898 fix non-root build script for ensure-libpq... +- ([#9959](https://github.com/EOSIO/eos/pull/9959)) merge back PR 9899, try using oob cmake so as to save building time +- ([#9970](https://github.com/EOSIO/eos/pull/9970)) Updating to the new Docker hub repo EOSIO instead EOS +- ([#9975](https://github.com/EOSIO/eos/pull/9975)) Release/2.1.x: Add additional contract to test_exhaustive_snapshot +- ([#9983](https://github.com/EOSIO/eos/pull/9983)) Add warning interval option for resource monitor plugin +- ([#9994](https://github.com/EOSIO/eos/pull/9994)) Add unit tests for new fields added for get account in PR#9838 +- ([#10014](https://github.com/EOSIO/eos/pull/10014)) [release 2.1.x] Fix LRT triggers +- ([#10020](https://github.com/EOSIO/eos/pull/10020)) revert changes to empty string as present for lower_bound, upper_bound,or index_value +- ([#10031](https://github.com/EOSIO/eos/pull/10031)) [release 2.1.x] Fix MacOS base image failures +- ([#10042](https://github.com/EOSIO/eos/pull/10042)) [release 2.1.x] Updated Mojave libpqxx dependency +- ([#10046](https://github.com/EOSIO/eos/pull/10046)) Reduce Docker Hub Manifest Queries +- ([#10054](https://github.com/EOSIO/eos/pull/10054)) Fix multiversion test failure - merge 2.1.x ## Documentation -- ([#10094](https://github.com/EOSIO/eos/pull/10094)) [docs] Add EOSIO 2.0.10 release notes to dev portal - 2.0 +- ([#9825](https://github.com/EOSIO/eos/pull/9825)) [docs] add how to: local testnet with consensus +- ([#9908](https://github.com/EOSIO/eos/pull/9908)) Add MacOS 10.15 (Catalina) to list of supported OSs in README +- ([#9914](https://github.com/EOSIO/eos/pull/9914)) [docs] add improvements based on code review +- ([#9921](https://github.com/EOSIO/eos/pull/9921)) [docs] 2.1.x local testnet with consensus +- ([#9925](https://github.com/EOSIO/eos/pull/9925)) [docs] cleos doc-a-thon feedback +- ([#9933](https://github.com/EOSIO/eos/pull/9933)) [docs] cleos doc-a-thon feedback 2 +- ([#9934](https://github.com/EOSIO/eos/pull/9934)) [docs] cleos doc-a-thon feedback 3 +- ([#9938](https://github.com/EOSIO/eos/pull/9938)) [docs] cleos doc-a-thon feedback 4 +- ([#9952](https://github.com/EOSIO/eos/pull/9952)) [docs] 2.1.x - improve annotation for db_update_i64 +- ([#10009](https://github.com/EOSIO/eos/pull/10009)) [docs] Update various cleos how-tos and fix index - 2.1 diff --git a/docs/index.md b/docs/index.md index 9cfcdb8292e..962eadf61a4 100644 --- a/docs/index.md +++ b/docs/index.md @@ -14,8 +14,7 @@ The basic relationship between these components is illustrated in the diagram be Additional EOSIO Resources: * [EOSIO Utilities](10_utilities/index.md) - Utilities that complement the EOSIO software. -* [Upgrade Guide](20_upgrade-guide/index.md) - EOSIO version/protocol upgrade guide. -* [Release Notes](30_release-notes/index.md) - All release notes for this EOSIO version. +* [Upgrade Guides](20_upgrade-guides/index.md) - EOSIO version/protocol upgrade guides. [[info | What's Next?]] | [Install the EOSIO Software](00_install/index.md) before exploring the sections above. diff --git a/eosio-wasm-spec-tests b/eosio-wasm-spec-tests index 25c46086d69..22f7f62d545 160000 --- a/eosio-wasm-spec-tests +++ b/eosio-wasm-spec-tests @@ -1 +1 @@ -Subproject commit 25c46086d694f5973ae1123e6a8aaa6e7232a2d4 +Subproject commit 22f7f62d5451ee57f14b2c3b9f62e35da50560f1 diff --git a/libraries/CMakeLists.txt b/libraries/CMakeLists.txt index d8b2168a2d7..59495bae32b 100644 --- a/libraries/CMakeLists.txt +++ b/libraries/CMakeLists.txt @@ -1,3 +1,18 @@ +option(PORTABLE CACHE ON) # rocksdb: don't use sse4.2 +option(WITH_GFLAGS CACHE OFF) # rocksdb: don't use gflags +option(WITH_TESTS CACHE OFF) # rocksdb: don't build this +option(WITH_TOOLS CACHE OFF) # rocksdb: don't build this +option(WITH_BENCHMARK_TOOLS CACHE OFF) # rocksdb: don't build this +option(FAIL_ON_WARNINGS CACHE OFF) # rocksdb: stop the madness: warnings change over time + +#on Linux, rocksdb will monkey with CMAKE_INSTALL_PREFIX is this is on +set(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT OFF) +# rocksdb disables USE_RTTI for release build, which breaks +# exception handling on MacOS. +if(APPLE) + set(USE_RTTI ON) +endif() + add_subdirectory( fc ) add_subdirectory( builtins ) add_subdirectory( softfloat ) @@ -7,13 +22,11 @@ add_subdirectory( appbase ) add_subdirectory( chain ) add_subdirectory( testing ) add_subdirectory( version ) - -#turn tools&tests off; not needed for library build -set(BUILD_TESTS OFF CACHE BOOL "Build GTest-based tests") -set(BUILD_TOOLS OFF CACHE BOOL "Build wabt tools") -set(RUN_RE2C OFF CACHE BOOL "Run re2c") -set(WITH_EXCEPTIONS ON CACHE BOOL "Build with exceptions enabled" FORCE) -add_subdirectory( wabt ) +add_subdirectory( state_history ) +add_subdirectory( abieos ) +add_subdirectory( rocksdb ) +add_subdirectory( chain_kv ) +add_subdirectory( se-helpers ) set(USE_EXISTING_SOFTFLOAT ON CACHE BOOL "use pre-exisiting softfloat lib") set(ENABLE_TOOLS OFF CACHE BOOL "Build tools") @@ -38,3 +51,8 @@ set_property(GLOBAL PROPERTY CTEST_CUSTOM_TESTS_IGNORE "change_authkey import_ed decrypt_ec decrypt_rsa ssh logs generate_rsa import_ec echo\ yubico_otp wrap_data wrap info import_rsa import_authkey generate_hmac generate_ec\ attest pbkdf2 parsing ${_CTEST_CUSTOM_TESTS_IGNORE}") + +find_package(OpenSSL REQUIRED) +option(AMQP-CPP_LINUX_TCP CACHE ON) +add_subdirectory( amqp-cpp EXCLUDE_FROM_ALL ) +target_include_directories(amqpcpp PRIVATE "${OPENSSL_INCLUDE_DIR}") diff --git a/libraries/abieos b/libraries/abieos new file mode 160000 index 00000000000..e5dad451c2b --- /dev/null +++ b/libraries/abieos @@ -0,0 +1 @@ +Subproject commit e5dad451c2b1cb58b0d53573ab98e5a8ed8455a6 diff --git a/libraries/amqp-cpp b/libraries/amqp-cpp new file mode 160000 index 00000000000..e7f76bc75d7 --- /dev/null +++ b/libraries/amqp-cpp @@ -0,0 +1 @@ +Subproject commit e7f76bc75d7855012450763a638092925d52f417 diff --git a/libraries/appbase b/libraries/appbase index 26957646dda..144b2e239d6 160000 --- a/libraries/appbase +++ b/libraries/appbase @@ -1 +1 @@ -Subproject commit 26957646dda6245ad97e205dae878cc3c65141cf +Subproject commit 144b2e239d6fd93a8336543bf9eda7c52ea8c77e diff --git a/libraries/chain/CMakeLists.txt b/libraries/chain/CMakeLists.txt index e5f9c9ef988..d4c1bbb6e43 100644 --- a/libraries/chain/CMakeLists.txt +++ b/libraries/chain/CMakeLists.txt @@ -18,23 +18,27 @@ else() endif() if("eos-vm-oc" IN_LIST EOSIO_WASM_RUNTIMES) - set(CHAIN_EOSVMOC_SOURCES webassembly/eos-vm-oc/code_cache.cpp - webassembly/eos-vm-oc/executor.cpp - webassembly/eos-vm-oc/memory.cpp - webassembly/eos-vm-oc/intrinsic.cpp - webassembly/eos-vm-oc/LLVMJIT.cpp - webassembly/eos-vm-oc/LLVMEmitIR.cpp - webassembly/eos-vm-oc/compile_monitor.cpp - webassembly/eos-vm-oc/compile_trampoline.cpp - webassembly/eos-vm-oc/ipc_helpers.cpp - webassembly/eos-vm-oc/gs_seg_helpers.c - webassembly/eos-vm-oc.cpp) + enable_language(ASM) + set(CHAIN_EOSVMOC_SOURCES webassembly/runtimes/eos-vm-oc/code_cache.cpp + webassembly/runtimes/eos-vm-oc/executor.cpp + webassembly/runtimes/eos-vm-oc/memory.cpp + webassembly/runtimes/eos-vm-oc/intrinsic.cpp + webassembly/runtimes/eos-vm-oc/LLVMJIT.cpp + webassembly/runtimes/eos-vm-oc/LLVMEmitIR.cpp + webassembly/runtimes/eos-vm-oc/compile_monitor.cpp + webassembly/runtimes/eos-vm-oc/compile_trampoline.cpp + webassembly/runtimes/eos-vm-oc/ipc_helpers.cpp + webassembly/runtimes/eos-vm-oc/gs_seg_helpers.c + webassembly/runtimes/eos-vm-oc/stack.cpp + webassembly/runtimes/eos-vm-oc/switch_stack_linux.s + webassembly/runtimes/eos-vm-oc.cpp + webassembly/runtimes/eos-vm-oc/default_real_main.cpp) if(LLVM_VERSION VERSION_LESS 7.1 AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang") enable_language(ASM-LLVMWAR) - list(APPEND CHAIN_EOSVMOC_SOURCES webassembly/eos-vm-oc/llvmWARshim.llvmwar) + list(APPEND CHAIN_EOSVMOC_SOURCES webassembly/runtimes/eos-vm-oc/llvmWARshim.llvmwar) else() - list(APPEND CHAIN_EOSVMOC_SOURCES webassembly/eos-vm-oc/llvmWARshim.cpp) + list(APPEND CHAIN_EOSVMOC_SOURCES webassembly/runtimes/eos-vm-oc/llvmWARshim.cpp) endif() llvm_map_components_to_libnames(LLVM_LIBS support core passes mcjit native orcjit) @@ -45,10 +49,30 @@ if("eos-vm-oc" IN_LIST EOSIO_WASM_RUNTIMES) endif() if("eos-vm" IN_LIST EOSIO_WASM_RUNTIMES OR "eos-vm-jit" IN_LIST EOSIO_WASM_RUNTIMES) - set(CHAIN_EOSVM_SOURCES "webassembly/eos-vm.cpp") + set(CHAIN_EOSVM_SOURCES "webassembly/runtimes/eos-vm.cpp") set(CHAIN_EOSVM_LIBRARIES eos-vm) endif() +set(CHAIN_WEBASSEMBLY_SOURCES + webassembly/action.cpp + webassembly/authorization.cpp + webassembly/cf_system.cpp + webassembly/cf_transaction.cpp + webassembly/compiler_builtins.cpp + webassembly/context_free.cpp + webassembly/console.cpp + webassembly/crypto.cpp + webassembly/database.cpp + webassembly/kv_database.cpp + webassembly/memory.cpp + webassembly/permission.cpp + webassembly/privileged.cpp + webassembly/producer.cpp + webassembly/softfloat.cpp + webassembly/system.cpp + webassembly/transaction.cpp +) + ## SORT .cpp by most likely to change / break compile add_library( eosio_chain merkle.cpp @@ -59,6 +83,7 @@ add_library( eosio_chain block_header_state.cpp block_state.cpp fork_database.cpp + combined_database.cpp controller.cpp authorization_manager.cpp resource_limits.cpp @@ -72,25 +97,21 @@ add_library( eosio_chain genesis_state.cpp ${CMAKE_CURRENT_BINARY_DIR}/genesis_state_root_key.cpp -# chain_config.cpp -# block_trace.cpp wast_to_wasm.cpp wasm_interface.cpp wasm_eosio_validation.cpp wasm_eosio_injection.cpp + wasm_config.cpp apply_context.cpp abi_serializer.cpp asset.cpp snapshot.cpp - webassembly/wabt.cpp ${CHAIN_EOSVMOC_SOURCES} ${CHAIN_EOSVM_SOURCES} + ${CHAIN_WEBASSEMBLY_SOURCES} -# get_config.cpp -# -# contracts/chain_initializer.cpp - + authority.cpp trace.cpp transaction_metadata.cpp protocol_state_object.cpp @@ -101,23 +122,32 @@ add_library( eosio_chain whitelisted_intrinsics.cpp thread_utils.cpp platform_timer_accuracy.cpp + backing_store/kv_context.cpp + backing_store/db_context.cpp + backing_store/db_context_chainbase.cpp + backing_store/db_context_rocksdb.cpp + backing_store/db_key_value_format.cpp + backing_store/db_key_value_any_lookup.cpp ${PLATFORM_TIMER_IMPL} ${HEADERS} ) target_link_libraries( eosio_chain fc chainbase Logging IR WAST WASM Runtime - softfloat builtins wabt ${CHAIN_EOSVM_LIBRARIES} ${LLVM_LIBS} ${CHAIN_RT_LINKAGE} + softfloat builtins rocksdb ${CHAIN_EOSVM_LIBRARIES} ${LLVM_LIBS} ${CHAIN_RT_LINKAGE} ) target_include_directories( eosio_chain PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/../wasm-jit/Include" "${CMAKE_CURRENT_SOURCE_DIR}/libraries/eos-vm/include" - "${CMAKE_SOURCE_DIR}/libraries/wabt" - "${CMAKE_BINARY_DIR}/libraries/wabt" + "${CMAKE_CURRENT_SOURCE_DIR}/../rocksdb/include" + "${CMAKE_CURRENT_SOURCE_DIR}/../chain_kv/include" ) +add_library(eosio_chain_wrap INTERFACE ) +target_link_libraries(eosio_chain_wrap INTERFACE eosio_chain) + if("eos-vm-oc" IN_LIST EOSIO_WASM_RUNTIMES) - target_link_libraries(eosio_chain "-Wl,-wrap=main") + target_link_libraries(eosio_chain_wrap INTERFACE "-Wl,-wrap=main") endif() foreach(RUNTIME ${EOSIO_WASM_RUNTIMES}) @@ -131,6 +161,8 @@ if(EOSVMOC_ENABLE_DEVELOPER_OPTIONS) target_compile_definitions(eosio_chain PUBLIC EOSIO_EOS_VM_OC_DEVELOPER) endif() +add_subdirectory ( backing_store/tests ) + install( TARGETS eosio_chain RUNTIME DESTINATION ${CMAKE_INSTALL_FULL_BINDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR} diff --git a/libraries/chain/abi_serializer.cpp b/libraries/chain/abi_serializer.cpp index 7f72e0f66cb..04bcaa8871b 100644 --- a/libraries/chain/abi_serializer.cpp +++ b/libraries/chain/abi_serializer.cpp @@ -38,7 +38,7 @@ namespace eosio { namespace chain { if( is_array ) fc::raw::pack( ds, var.as>() ); else if ( is_optional ) - fc::raw::pack( ds, var.as>() ); + fc::raw::pack( ds, var.as>() ); else fc::raw::pack( ds, var.as()); }; @@ -51,7 +51,7 @@ namespace eosio { namespace chain { if( is_array ) return variant_from_stream>(stream); else if ( is_optional ) - return variant_from_stream>(stream); + return variant_from_stream>(stream); return variant_from_stream(stream); }, pack_function() @@ -65,7 +65,7 @@ namespace eosio { namespace chain { if( is_array ) return variant_from_stream>(stream); else if ( is_optional ) - return variant_from_stream>(stream); + return variant_from_stream>(stream); return variant_from_stream(stream, yield); }, pack_function() @@ -77,11 +77,6 @@ namespace eosio { namespace chain { set_abi(abi, yield); } - abi_serializer::abi_serializer( const abi_def& abi, const fc::microseconds& max_serialization_time) { - configure_built_in_types(); - set_abi(abi, max_serialization_time); - } - void abi_serializer::add_specialized_unpack_pack( const string& name, std::pair unpack_pack ) { built_in_types[name] = std::move( unpack_pack ); @@ -139,8 +134,10 @@ namespace eosio { namespace chain { structs.clear(); actions.clear(); tables.clear(); + kv_tables.clear(); error_messages.clear(); variants.clear(); + action_results.clear(); for( const auto& st : abi.structs ) structs[st.name] = st; @@ -157,12 +154,18 @@ namespace eosio { namespace chain { for( const auto& t : abi.tables ) tables[t.name] = t.type; + for( const auto& kt : abi.kv_tables.value ) + kv_tables[kt.first] = kt.second; + for( const auto& e : abi.error_messages ) error_messages[e.error_code] = e.error_msg; for( const auto& v : abi.variants.value ) variants[v.name] = v; + for( const auto& r : abi.action_results.value ) + action_results[r.name] = r.result_type; + /** * The ABI vector may contain duplicates which would make it * an invalid ABI @@ -171,16 +174,14 @@ namespace eosio { namespace chain { EOS_ASSERT( structs.size() == abi.structs.size(), duplicate_abi_struct_def_exception, "duplicate struct definition detected" ); EOS_ASSERT( actions.size() == abi.actions.size(), duplicate_abi_action_def_exception, "duplicate action definition detected" ); EOS_ASSERT( tables.size() == abi.tables.size(), duplicate_abi_table_def_exception, "duplicate table definition detected" ); + EOS_ASSERT( kv_tables.size() == abi.kv_tables.value.size(), duplicate_abi_kv_table_def_exception, "duplicate kv table definition detected" ); EOS_ASSERT( error_messages.size() == abi.error_messages.size(), duplicate_abi_err_msg_def_exception, "duplicate error message definition detected" ); EOS_ASSERT( variants.size() == abi.variants.value.size(), duplicate_abi_variant_def_exception, "duplicate variant definition detected" ); + EOS_ASSERT( action_results.size() == abi.action_results.value.size(), duplicate_abi_action_results_def_exception, "duplicate action results definition detected" ); validate(ctx); } - void abi_serializer::set_abi(const abi_def& abi, const fc::microseconds& max_serialization_time) { - return set_abi(abi, create_yield_function(max_serialization_time)); - } - bool abi_serializer::is_builtin_type(const std::string_view& type)const { return built_in_types.find(type) != built_in_types.end(); } @@ -215,8 +216,8 @@ namespace eosio { namespace chain { return _is_type(type, ctx); } - bool abi_serializer::is_type(const std::string_view& type, const fc::microseconds& max_serialization_time) const { - return is_type(type, create_yield_function(max_serialization_time)); + bool abi_serializer::is_kv_table(const std::string_view& type)const { + return kv_tables.find(name(type)) != kv_tables.end(); } std::string_view abi_serializer::fundamental_type(const std::string_view& type)const { @@ -243,6 +244,9 @@ namespace eosio { namespace chain { if( typedefs.find(type) != typedefs.end() ) return _is_type(typedefs.find(type)->second, ctx); if( structs.find(type) != structs.end() ) return true; if( variants.find(type) != variants.end() ) return true; + if( eosio::chain::is_string_valid_name(type) ) { + if( kv_tables.find(name(type)) != kv_tables.end() ) return true; + } return false; } @@ -301,6 +305,18 @@ namespace eosio { namespace chain { ctx.check_deadline(); EOS_ASSERT(_is_type(t.second, ctx), invalid_type_inside_abi, "${type}", ("type",impl::limit_size(t.second)) ); } FC_CAPTURE_AND_RETHROW( (t) ) } + + for( const auto& kt : kv_tables ) { + ctx.check_deadline(); + EOS_ASSERT(_is_type(kt.second.type, ctx), invalid_type_inside_abi, + "Invalid reference in struct ${type}", ("type", impl::limit_size(kt.second.type))); + EOS_ASSERT( !kt.second.primary_index.type.empty(), invalid_type_inside_abi, "missing primary index$ {p}", ("p",impl::limit_size(kt.first.to_string()))); + } + + for( const auto& r : action_results ) { try { + ctx.check_deadline(); + EOS_ASSERT(_is_type(r.second, ctx), invalid_type_inside_abi, "${type}", ("type",impl::limit_size(r.second)) ); + } FC_CAPTURE_AND_RETHROW( (r) ) } } std::string_view abi_serializer::resolve_type(const std::string_view& type)const { @@ -394,7 +410,7 @@ namespace eosio { namespace chain { } else { auto v_itr = variants.find(rtype); if( v_itr != variants.end() ) { - ctx.hint_variant_type_if_in_array( v_itr ); + ctx.hint_variant_type_if_in_array(v_itr); fc::unsigned_int select; try { fc::raw::unpack(stream, select); @@ -404,6 +420,13 @@ namespace eosio { namespace chain { auto h1 = ctx.push_to_path( impl::variant_path_item{ .variant_itr = v_itr, .variant_ordinal = static_cast(select) } ); return vector{v_itr->second.types[select], _binary_to_variant(v_itr->second.types[select], stream, ctx)}; } + + if( !kv_tables.empty() && is_string_valid_name(rtype) ) { + if( auto kv_itr = kv_tables.find(name(rtype)); kv_itr != kv_tables.end() ) { + auto &kv_table = kv_itr->second; + return _binary_to_variant(kv_table.type, stream, ctx); + } + } } fc::mutable_variant_object mvo; @@ -426,20 +449,12 @@ namespace eosio { namespace chain { return _binary_to_variant(type, binary, ctx); } - fc::variant abi_serializer::binary_to_variant( const std::string_view& type, const bytes& binary, const fc::microseconds& max_serialization_time, bool short_path )const { - return binary_to_variant( type, binary, create_yield_function(max_serialization_time), short_path ); - } - fc::variant abi_serializer::binary_to_variant( const std::string_view& type, fc::datastream& binary, const yield_function_t& yield, bool short_path )const { impl::binary_to_variant_context ctx(*this, yield, type); ctx.short_path = short_path; return _binary_to_variant(type, binary, ctx); } - fc::variant abi_serializer::binary_to_variant( const std::string_view& type, fc::datastream& binary, const fc::microseconds& max_serialization_time, bool short_path )const { - return binary_to_variant( type, binary, create_yield_function(max_serialization_time), short_path ); - } - void abi_serializer::_variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream& ds, impl::variant_to_binary_context& ctx )const { try { auto h = ctx.enter_scope(); @@ -540,6 +555,15 @@ namespace eosio { namespace chain { } else { EOS_THROW( pack_exception, "Unexpected input encountered while processing struct '${p}'", ("p",ctx.get_path_string()) ); } + } else if( var.is_object() ) { + if( !kv_tables.empty() && is_string_valid_name(rtype) ) { + if( auto kv_itr = kv_tables.find(name(rtype));kv_itr != kv_tables.end() ) { + auto& kv_table = kv_itr->second; + _variant_to_binary( kv_table.type, var, ds, ctx ); + } + } else { + EOS_THROW(invalid_type_inside_abi, "Unknown type ${type}", ("type", ctx.maybe_shorten(type))); + } } else { EOS_THROW( invalid_type_inside_abi, "Unknown type ${type}", ("type",ctx.maybe_shorten(type)) ); } @@ -565,20 +589,12 @@ namespace eosio { namespace chain { return _variant_to_binary(type, var, ctx); } - bytes abi_serializer::variant_to_binary( const std::string_view& type, const fc::variant& var, const fc::microseconds& max_serialization_time, bool short_path ) const { - return variant_to_binary( type, var, create_yield_function(max_serialization_time), short_path ); - } - void abi_serializer::variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream& ds, const yield_function_t& yield, bool short_path )const { impl::variant_to_binary_context ctx(*this, yield, type); ctx.short_path = short_path; _variant_to_binary(type, var, ds, ctx); } - void abi_serializer::variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream& ds, const fc::microseconds& max_serialization_time, bool short_path ) const { - variant_to_binary( type, var, create_yield_function(max_serialization_time), short_path ); - } - type_name abi_serializer::get_action_type(name action)const { auto itr = actions.find(action); if( itr != actions.end() ) return itr->second; @@ -590,10 +606,23 @@ namespace eosio { namespace chain { return type_name(); } - optional abi_serializer::get_error_message( uint64_t error_code )const { + type_name abi_serializer::get_kv_table_type(name action)const { + if( auto itr = kv_tables.find(action);itr != kv_tables.end() ) + return itr->second.type; + + return type_name(); + } + + type_name abi_serializer::get_action_result_type(name action_result)const { + auto itr = action_results.find(action_result); + if( itr != action_results.end() ) return itr->second; + return type_name(); + } + + std::optional abi_serializer::get_error_message( uint64_t error_code )const { auto itr = error_messages.find( error_code ); if( itr == error_messages.end() ) - return optional(); + return std::optional(); return itr->second; } @@ -646,30 +675,30 @@ namespace eosio { namespace chain { auto& b = path.back(); - EOS_ASSERT( b.contains(), abi_exception, "trying to set array index without first pushing new array index item" ); + EOS_ASSERT( std::holds_alternative(b), abi_exception, "trying to set array index without first pushing new array index item" ); - b.get().array_index = i; + std::get(b).array_index = i; } void abi_traverse_context_with_path::hint_array_type_if_in_array() { - if( path.size() == 0 || !path.back().contains() ) + if( path.size() == 0 || !std::holds_alternative(path.back()) ) return; - path.back().get().type_hint = array_type_path_root{}; + std::get(path.back()).type_hint = array_type_path_root{}; } void abi_traverse_context_with_path::hint_struct_type_if_in_array( const map::const_iterator& itr ) { - if( path.size() == 0 || !path.back().contains() ) + if( path.size() == 0 || !std::holds_alternative(path.back()) ) return; - path.back().get().type_hint = struct_type_path_root{ .struct_itr = itr }; + std::get(path.back()).type_hint = struct_type_path_root{ .struct_itr = itr }; } void abi_traverse_context_with_path::hint_variant_type_if_in_array( const map::const_iterator& itr ) { - if( path.size() == 0 || !path.back().contains() ) + if( path.size() == 0 || !std::holds_alternative(path.back()) ) return; - path.back().get().type_hint = variant_type_path_root{ .variant_itr = itr }; + std::get(path.back()).type_hint = variant_type_path_root{ .variant_itr = itr }; } constexpr size_t const_strlen( const char* str ) @@ -789,13 +818,13 @@ namespace eosio { namespace chain { void operator()( const array_index_path_item& item ) { const auto& th = item.type_hint; - if( th.contains() ) { - const auto& str = th.get().struct_itr->first; + if( std::holds_alternative(th) ) { + const auto& str = std::get(th).struct_itr->first; output_name( s, str, shorten_names ); - } else if( th.contains() ) { - const auto& str = th.get().variant_itr->first; + } else if( std::holds_alternative(th) ) { + const auto& str = std::get(th).variant_itr->first; output_name( s, str, shorten_names ); - } else if( th.contains() ) { + } else if( std::holds_alternative(th) ) { s << "ARRAY"; } else { s << "UNKNOWN"; @@ -819,21 +848,21 @@ namespace eosio { namespace chain { generate_path_string_visitor visitor(shorten_names, !full_path); if( full_path ) - root_of_path.visit( visitor ); + std::visit( visitor, root_of_path ); for( size_t i = 0, n = path.size(); i < n; ++i ) { - if( full_path && !path[i].contains() ) + if( full_path && !std::holds_alternative(path[i]) ) visitor.add_dot(); - path[i].visit( visitor ); + std::visit( visitor, path[i] ); } if( !full_path ) { - if( visitor.last_path_item.contains() ) { - root_of_path.visit( visitor ); + if( std::holds_alternative(visitor.last_path_item) ) { + std::visit( visitor, root_of_path ); } else { path_item_type_visitor vis2(visitor.s, shorten_names); - visitor.last_path_item.visit(vis2); + std::visit(vis2, visitor.last_path_item); } } diff --git a/libraries/chain/apply_context.cpp b/libraries/chain/apply_context.cpp index df192fa0fa1..741a5a5c550 100644 --- a/libraries/chain/apply_context.cpp +++ b/libraries/chain/apply_context.cpp @@ -11,11 +11,14 @@ #include #include #include +#include using boost::container::flat_set; namespace eosio { namespace chain { +using db_context = backing_store::db_context; + static inline void print_debug(account_name receiver, const action_trace& ar) { if (!ar.console.empty()) { auto prefix = fc::format_string( @@ -43,80 +46,122 @@ apply_context::apply_context(controller& con, transaction_context& trx_ctx, uint ,idx_double(*this) ,idx_long_double(*this) { + kv_iterators.emplace_back(); // the iterator handle with value 0 is reserved action_trace& trace = trx_ctx.get_action_trace(action_ordinal); act = &trace.act; receiver = trace.receiver; context_free = trace.context_free; + _db_context = control.kv_db().create_db_context(*this, receiver); +} + +template +void apply_context::check_unprivileged_resource_usage(const char* resource, const flat_set& deltas) { + const size_t checktime_interval = 10; + const bool not_in_notify_context = (receiver == act->account); + size_t counter = 0; + for (const auto& entry : deltas) { + if (counter == checktime_interval) { + trx_context.checktime(); + counter = 0; + } + if (entry.delta > 0 && entry.account != receiver) { + EOS_ASSERT(not_in_notify_context, Exception, + "unprivileged contract cannot increase ${resource} usage of another account within a notify context: " + "${account}", + ("resource", resource) + ("account", entry.account)); + EOS_ASSERT(has_authorization(entry.account), Exception, + "unprivileged contract cannot increase ${resource} usage of another account that has not authorized the " + "action: ${account}", + ("resource", resource) + ("account", entry.account)); + } + ++counter; + } } void apply_context::exec_one() { auto start = fc::time_point::now(); - action_receipt r; - r.receiver = receiver; - r.act_digest = digest_type::hash(*act); + digest_type act_digest; const auto& cfg = control.get_global_properties().configuration; const account_metadata_object* receiver_account = nullptr; + + auto handle_exception = [&](const auto& e) + { + action_trace& trace = trx_context.get_action_trace( action_ordinal ); + trace.error_code = controller::convert_exception_to_error_code( e ); + trace.except = e; + finalize_trace( trace, start ); + throw; + }; + try { try { + action_return_value.clear(); + kv_iterators.resize(1); + kv_destroyed_iterators.clear(); + if (!context_free) { + kv_backing_store = control.kv_db().create_kv_context(receiver, create_kv_resource_manager(*this), control.get_global_properties().kv_configuration); + } receiver_account = &db.get( receiver ); - privileged = receiver_account->is_privileged(); - auto native = control.find_apply_handler( receiver, act->account, act->name ); - if( native ) { - if( trx_context.enforce_whiteblacklist && control.is_producing_block() ) { - control.check_contract_list( receiver ); - control.check_action_list( act->account, act->name ); + if( !(context_free && control.skip_trx_checks()) ) { + privileged = receiver_account->is_privileged(); + auto native = control.find_apply_handler( receiver, act->account, act->name ); + if( native ) { + if( trx_context.enforce_whiteblacklist && control.is_producing_block() ) { + control.check_contract_list( receiver ); + control.check_action_list( act->account, act->name ); + } + (*native)( *this ); } - (*native)( *this ); - } - if( ( receiver_account->code_hash != digest_type() ) && - ( !( act->account == config::system_account_name - && act->name == N( setcode ) - && receiver == config::system_account_name ) - || control.is_builtin_activated( builtin_protocol_feature_t::forward_setcode ) - ) - ) { - if( trx_context.enforce_whiteblacklist && control.is_producing_block() ) { - control.check_contract_list( receiver ); - control.check_action_list( act->account, act->name ); + if( ( receiver_account->code_hash != digest_type() ) && + ( !( act->account == config::system_account_name + && act->name == "setcode"_n + && receiver == config::system_account_name ) + || control.is_builtin_activated( builtin_protocol_feature_t::forward_setcode ) + ) + ) { + if( trx_context.enforce_whiteblacklist && control.is_producing_block() ) { + control.check_contract_list( receiver ); + control.check_action_list( act->account, act->name ); + } + try { + control.get_wasm_interface().apply( receiver_account->code_hash, receiver_account->vm_type, receiver_account->vm_version, *this ); + } catch( const wasm_exit& ) {} } - try { - control.get_wasm_interface().apply( receiver_account->code_hash, receiver_account->vm_type, receiver_account->vm_version, *this ); - } catch( const wasm_exit& ) {} - } - if( !privileged && control.is_builtin_activated( builtin_protocol_feature_t::ram_restrictions ) ) { - const size_t checktime_interval = 10; - size_t counter = 0; - bool not_in_notify_context = (receiver == act->account); - const auto end = _account_ram_deltas.end(); - for( auto itr = _account_ram_deltas.begin(); itr != end; ++itr, ++counter ) { - if( counter == checktime_interval ) { - trx_context.checktime(); - counter = 0; - } - if( itr->delta > 0 && itr->account != receiver ) { - EOS_ASSERT( not_in_notify_context, unauthorized_ram_usage_increase, - "unprivileged contract cannot increase RAM usage of another account within a notify context: ${account}", - ("account", itr->account) - ); - EOS_ASSERT( has_authorization( itr->account ), unauthorized_ram_usage_increase, - "unprivileged contract cannot increase RAM usage of another account that has not authorized the action: ${account}", - ("account", itr->account) - ); + if (!privileged) { + if (control.is_builtin_activated(builtin_protocol_feature_t::ram_restrictions)) { + check_unprivileged_resource_usage("RAM", _account_ram_deltas); } } } } FC_RETHROW_EXCEPTIONS( warn, "pending console output: ${console}", ("console", _pending_console_output) ) - } catch( const fc::exception& e ) { - action_trace& trace = trx_context.get_action_trace( action_ordinal ); - trace.error_code = controller::convert_exception_to_error_code( e ); - trace.except = e; - finalize_trace( trace, start ); + + if( control.is_builtin_activated( builtin_protocol_feature_t::action_return_value ) ) { + act_digest = generate_action_digest( + [this](const char* data, uint32_t datalen) { + return trx_context.hash_with_checktime(data, datalen); + }, + *act, + action_return_value + ); + } else { + act_digest = digest_type::hash(*act); + } + } catch ( const std::bad_alloc& ) { throw; + } catch ( const boost::interprocess::bad_alloc& ) { + throw; + } catch( const fc::exception& e ) { + handle_exception(e); + } catch ( const std::exception& e ) { + auto wrapper = fc::std_exception_wrapper::from_current_exception(e); + handle_exception(wrapper); } // Note: It should not be possible for receiver_account to be invalidated because: @@ -124,6 +169,15 @@ void apply_context::exec_one() // * a pointer to an object in a chainbase index is not invalidated if the fields of that object are modified; // * and, the *receiver_account object itself cannot be removed because accounts cannot be deleted in EOSIO. + action_trace& trace = trx_context.get_action_trace( action_ordinal ); + trace.return_value = std::move(action_return_value); + trace.receipt.emplace(); + + action_receipt& r = *trace.receipt; + r.receiver = receiver; + r.act_digest = act_digest; + + r.global_sequence = next_global_sequence(); r.recv_sequence = next_recv_sequence( *receiver_account ); @@ -141,10 +195,7 @@ void apply_context::exec_one() r.auth_sequence[auth.actor] = next_auth_sequence( auth.actor ); } - action_trace& trace = trx_context.get_action_trace( action_ordinal ); - trace.receipt = r; - - trx_context.executed.emplace_back( std::move(r) ); + trx_context.executed_action_receipt_digests.emplace_back( r.digest() ); finalize_trace( trace, start ); @@ -168,9 +219,12 @@ void apply_context::exec() { _notified.emplace_back( receiver, action_ordinal ); exec_one(); + increment_action_id(); for( uint32_t i = 1; i < _notified.size(); ++i ) { std::tie( receiver, action_ordinal ) = _notified[i]; + _db_context->receiver = receiver; exec_one(); + increment_action_id(); } if( _cfa_inline_actions.size() > 0 || _inline_actions.size() > 0 ) { @@ -192,7 +246,7 @@ bool apply_context::is_account( const account_name& account )const { return nullptr != db.find( account ); } -void apply_context::require_authorization( const account_name& account ) { +void apply_context::require_authorization( const account_name& account ) const { for( uint32_t i=0; i < act->authorization.size(); i++ ) { if( act->authorization[i].actor == account ) { return; @@ -209,7 +263,7 @@ bool apply_context::has_authorization( const account_name& account )const { } void apply_context::require_authorization(const account_name& account, - const permission_name& permission) { + const permission_name& permission) const { for( uint32_t i=0; i < act->authorization.size(); i++ ) if( act->authorization[i].actor == account ) { if( act->authorization[i].permission == permission ) { @@ -233,6 +287,12 @@ void apply_context::require_recipient( account_name recipient ) { recipient, schedule_action( action_ordinal, recipient, false ) ); + + if (auto dm_logger = control.get_deep_mind_logger()) { + fc_dlog(*dm_logger, "CREATION_OP NOTIFY ${action_id}", + ("action_id", get_action_id()) + ); + } } } @@ -333,6 +393,12 @@ void apply_context::execute_inline( action&& a ) { _inline_actions.emplace_back( schedule_action( std::move(a), inline_receiver, false ) ); + + if (auto dm_logger = control.get_deep_mind_logger()) { + fc_dlog(*dm_logger, "CREATION_OP INLINE ${action_id}", + ("action_id", get_action_id()) + ); + } } void apply_context::execute_context_free_inline( action&& a ) { @@ -354,6 +420,12 @@ void apply_context::execute_context_free_inline( action&& a ) { _cfa_inline_actions.emplace_back( schedule_action( std::move(a), inline_receiver, true ) ); + + if (auto dm_logger = control.get_deep_mind_logger()) { + fc_dlog(*dm_logger, "CREATION_OP CFA_INLINE ${action_id}", + ("action_id", get_action_id()) + ); + } } @@ -373,7 +445,7 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a "only the deferred_transaction_generation_context extension is currently supported for deferred transactions" ); - const auto& context = itr->second.get(); + const auto& context = std::get(itr->second); EOS_ASSERT( context.sender == receiver, ill_formed_deferred_transaction_generation_context, "deferred transaction generaction context contains mismatching sender", @@ -383,15 +455,15 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a "deferred transaction generaction context contains mismatching sender_id", ("expected", sender_id)("actual", context.sender_id) ); - EOS_ASSERT( context.sender_trx_id == trx_context.id, ill_formed_deferred_transaction_generation_context, + EOS_ASSERT( context.sender_trx_id == trx_context.packed_trx.id(), ill_formed_deferred_transaction_generation_context, "deferred transaction generaction context contains mismatching sender_trx_id", - ("expected", trx_context.id)("actual", context.sender_trx_id) + ("expected", trx_context.packed_trx.id())("actual", context.sender_trx_id) ); } else { emplace_extension( trx.transaction_extensions, deferred_transaction_generation_context::extension_id(), - fc::raw::pack( deferred_transaction_generation_context( trx_context.id, sender_id, receiver ) ) + fc::raw::pack( deferred_transaction_generation_context( trx_context.packed_trx.id(), sender_id, receiver ) ) ); } trx.expiration = time_point_sec(); @@ -479,6 +551,8 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a } uint32_t trx_size = 0; + std::string event_id; + const char* operation = ""; if ( auto ptr = db.find(boost::make_tuple(receiver, sender_id)) ) { EOS_ASSERT( replace_existing, deferred_tx_duplicate, "deferred transaction with the same sender_id and payer already exists" ); @@ -489,11 +563,16 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a subjective_block_production_exception, "Replacing a deferred transaction is temporarily disabled." ); + if (control.get_deep_mind_logger() != nullptr) { + event_id = STORAGE_EVENT_ID("${id}", ("id", ptr->id)); + } + uint64_t orig_trx_ram_bytes = config::billable_size_v + ptr->packed_trx.size(); if( replace_deferred_activated ) { - add_ram_usage( ptr->payer, -static_cast( orig_trx_ram_bytes ) ); + // avoiding moving event_id to make logic easier to maintain + add_ram_usage( ptr->payer, -static_cast( orig_trx_ram_bytes ), storage_usage_trace(get_action_id(), std::string(event_id), "deferred_trx", "cancel", "deferred_trx_cancel") ); } else { - control.add_to_ram_correction( ptr->payer, orig_trx_ram_bytes ); + control.add_to_ram_correction( ptr->payer, orig_trx_ram_bytes, get_action_id(), event_id.c_str() ); } transaction_id_type trx_id_for_new_obj; @@ -503,8 +582,23 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a trx_id_for_new_obj = ptr->trx_id; } + if (auto dm_logger = control.get_deep_mind_logger()) { + fc_dlog(*dm_logger, "DTRX_OP MODIFY_CANCEL ${action_id} ${sender} ${sender_id} ${payer} ${published} ${delay} ${expiration} ${trx_id} ${trx}", + ("action_id", get_action_id()) + ("sender", receiver) + ("sender_id", sender_id) + ("payer", ptr->payer) + ("published", ptr->published) + ("delay", ptr->delay_until) + ("expiration", ptr->expiration) + ("trx_id", ptr->trx_id) + ("trx", fc::to_hex(ptr->packed_trx.data(), ptr->packed_trx.size())) + ); + } + // Use remove and create rather than modify because mutating the trx_id field in a modifier is unsafe. db.remove( *ptr ); + db.create( [&]( auto& gtx ) { gtx.trx_id = trx_id_for_new_obj; gtx.sender = receiver; @@ -515,6 +609,23 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a gtx.expiration = gtx.delay_until + fc::seconds(control.get_global_properties().configuration.deferred_trx_expiration_window); trx_size = gtx.set( trx ); + + if (auto dm_logger = control.get_deep_mind_logger()) { + operation = "update"; + event_id = STORAGE_EVENT_ID("${id}", ("id", gtx.id)); + + fc_dlog(*dm_logger, "DTRX_OP MODIFY_CREATE ${action_id} ${sender} ${sender_id} ${payer} ${published} ${delay} ${expiration} ${trx_id} ${trx}", + ("action_id", get_action_id()) + ("sender", receiver) + ("sender_id", sender_id) + ("payer", payer) + ("published", gtx.published) + ("delay", gtx.delay_until) + ("expiration", gtx.expiration) + ("trx_id", trx.id()) + ("trx", fc::to_hex(gtx.packed_trx.data(), gtx.packed_trx.size())) + ); + } } ); } else { db.create( [&]( auto& gtx ) { @@ -527,6 +638,23 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a gtx.expiration = gtx.delay_until + fc::seconds(control.get_global_properties().configuration.deferred_trx_expiration_window); trx_size = gtx.set( trx ); + + if (auto dm_logger = control.get_deep_mind_logger()) { + operation = "add"; + event_id = STORAGE_EVENT_ID("${id}", ("id", gtx.id)); + + fc_dlog(*dm_logger, "DTRX_OP CREATE ${action_id} ${sender} ${sender_id} ${payer} ${published} ${delay} ${expiration} ${trx_id} ${trx}", + ("action_id", get_action_id()) + ("sender", receiver) + ("sender_id", sender_id) + ("payer", payer) + ("published", gtx.published) + ("delay", gtx.delay_until) + ("expiration", gtx.expiration) + ("trx_id", gtx.trx_id) + ("trx", fc::to_hex(gtx.packed_trx.data(), gtx.packed_trx.size())) + ); + } } ); } @@ -536,14 +664,33 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a subjective_block_production_exception, "Cannot charge RAM to other accounts during notify." ); - add_ram_usage( payer, (config::billable_size_v + trx_size) ); + add_ram_usage( payer, (config::billable_size_v + trx_size), storage_usage_trace(get_action_id(), std::move(event_id), "deferred_trx", operation, "deferred_trx_add") ); } bool apply_context::cancel_deferred_transaction( const uint128_t& sender_id, account_name sender ) { + + auto& generated_transaction_idx = db.get_mutable_index(); const auto* gto = db.find(boost::make_tuple(sender, sender_id)); if ( gto ) { - add_ram_usage( gto->payer, -(config::billable_size_v + gto->packed_trx.size()) ); + std::string event_id; + if (auto dm_logger = control.get_deep_mind_logger()) { + event_id = STORAGE_EVENT_ID("${id}", ("id", gto->id)); + + fc_dlog(*dm_logger, "DTRX_OP CANCEL ${action_id} ${sender} ${sender_id} ${payer} ${published} ${delay} ${expiration} ${trx_id} ${trx}", + ("action_id", get_action_id()) + ("sender", receiver) + ("sender_id", sender_id) + ("payer", gto->payer) + ("published", gto->published) + ("delay", gto->delay_until) + ("expiration", gto->expiration) + ("trx_id", gto->trx_id) + ("trx", fc::to_hex(gto->packed_trx.data(), gto->packed_trx.size())) + ); + } + + add_ram_usage( gto->payer, -(config::billable_size_v + gto->packed_trx.size()), storage_usage_trace(get_action_id(), std::move(event_id), "deferred_trx", "cancel", "deferred_trx_cancel") ); generated_transaction_idx.remove(*gto); } return gto; @@ -579,18 +726,37 @@ const table_id_object& apply_context::find_or_create_table( name code, name scop return *existing_tid; } - update_db_usage(payer, config::billable_size_v); + std::string event_id; + if (control.get_deep_mind_logger() != nullptr) { + event_id = db_context::table_event(code, scope, table); + } + + update_db_usage(payer, config::billable_size_v, db_context::add_table_trace(get_action_id(), std::move(event_id))); return db.create([&](table_id_object &t_id){ t_id.code = code; t_id.scope = scope; t_id.table = table; t_id.payer = payer; + + if (auto dm_logger = control.get_deep_mind_logger()) { + db_context::log_insert_table(*dm_logger, get_action_id(), code, scope, table, payer); + } }); } void apply_context::remove_table( const table_id_object& tid ) { - update_db_usage(tid.payer, - config::billable_size_v); + std::string event_id; + if (control.get_deep_mind_logger() != nullptr) { + event_id = db_context::table_event(tid.code, tid.scope, tid.table); + } + + update_db_usage(tid.payer, - config::billable_size_v, db_context::rem_table_trace(get_action_id(), std::move(event_id)) ); + + if (auto dm_logger = control.get_deep_mind_logger()) { + db_context::log_remove_table(*dm_logger, get_action_id(), tid.code, tid.scope, tid.table, tid.payer); + } + db.remove(tid); } @@ -604,12 +770,7 @@ vector apply_context::get_active_producers() const { return accounts; } -bytes apply_context::get_packed_transaction() { - auto r = fc::raw::pack( static_cast(trx_context.trx) ); - return r; -} - -void apply_context::update_db_usage( const account_name& payer, int64_t delta ) { +void apply_context::update_db_usage( const account_name& payer, int64_t delta, const storage_usage_trace& trace ) { if( delta > 0 ) { if( !(privileged || payer == account_name(receiver) || control.is_builtin_activated( builtin_protocol_feature_t::ram_restrictions ) ) ) @@ -619,13 +780,13 @@ void apply_context::update_db_usage( const account_name& payer, int64_t delta ) require_authorization( payer ); } } - add_ram_usage(payer, delta); + add_ram_usage(payer, delta, trace); } int apply_context::get_action( uint32_t type, uint32_t index, char* buffer, size_t buffer_size )const { - const auto& trx = trx_context.trx; + const auto& trx = trx_context.packed_trx.get_transaction(); const action* act_ptr = nullptr; if( type == 0 ) { @@ -651,26 +812,43 @@ int apply_context::get_action( uint32_t type, uint32_t index, char* buffer, size int apply_context::get_context_free_data( uint32_t index, char* buffer, size_t buffer_size )const { - const auto& trx = trx_context.trx; + const packed_transaction::prunable_data_type::prunable_data_t& data = trx_context.packed_trx.get_prunable_data().prunable_data; + const bytes* cfd = nullptr; + if( std::holds_alternative(data) ) { + } else if( std::holds_alternative(data) ) { + if( index >= std::get(data).context_free_segments.size() ) return -1; + + cfd = trx_context.packed_trx.get_context_free_data(index); + } else { + const std::vector& context_free_data = + std::holds_alternative(data) ? + std::get(data).context_free_segments : + std::get(data).context_free_segments; + if( index >= context_free_data.size() ) return -1; - if( index >= trx.context_free_data.size() ) return -1; + cfd = &context_free_data[index]; + } - auto s = trx.context_free_data[index].size(); + if( !cfd ) { + if( control.is_producing_block() ) { + EOS_THROW( subjective_block_production_exception, "pruned context free data not available" ); + } else { + EOS_THROW( pruned_context_free_data_bad_block_exception, "pruned context free data not available" ); + } + } + + auto s = cfd->size(); if( buffer_size == 0 ) return s; auto copy_size = std::min( buffer_size, s ); - memcpy( buffer, trx.context_free_data[index].data(), copy_size ); + memcpy( buffer, cfd->data(), copy_size ); return copy_size; } -int apply_context::db_store_i64( name scope, name table, const account_name& payer, uint64_t id, const char* buffer, size_t buffer_size ) { - return db_store_i64( receiver, scope, table, payer, id, buffer, buffer_size); -} - -int apply_context::db_store_i64( name code, name scope, name table, const account_name& payer, uint64_t id, const char* buffer, size_t buffer_size ) { +int apply_context::db_store_i64_chainbase( name scope, name table, const account_name& payer, uint64_t id, const char* buffer, size_t buffer_size ) { // require_write_lock( scope ); - const auto& tab = find_or_create_table( code, scope, table, payer ); + const auto& tab = find_or_create_table( receiver, scope, table, payer ); auto tableid = tab.id; EOS_ASSERT( payer != account_name(), invalid_table_payer, "must specify a valid account to pay for new record" ); @@ -687,16 +865,26 @@ int apply_context::db_store_i64( name code, name scope, name table, const accoun }); int64_t billable_size = (int64_t)(buffer_size + config::billable_size_v); - update_db_usage( payer, billable_size); - keyval_cache.cache_table( tab ); - return keyval_cache.add( obj ); + std::string event_id; + if (control.get_deep_mind_logger() != nullptr) { + event_id = db_context::table_event(tab.code, tab.scope, tab.table, name(obj.primary_key)); + } + + update_db_usage( payer, billable_size, db_context::row_add_trace(get_action_id(), std::move(event_id)) ); + + if (auto dm_logger = control.get_deep_mind_logger()) { + db_context::log_row_insert(*dm_logger, get_action_id(), tab.code, tab.scope, tab.table, payer, name(obj.primary_key), buffer, buffer_size); + } + + db_iter_store.cache_table( tab ); + return db_iter_store.add( obj ); } -void apply_context::db_update_i64( int iterator, account_name payer, const char* buffer, size_t buffer_size ) { - const key_value_object& obj = keyval_cache.get( iterator ); +void apply_context::db_update_i64_chainbase( int iterator, account_name payer, const char* buffer, size_t buffer_size ) { + const key_value_object& obj = db_iter_store.get( iterator ); - const auto& table_obj = keyval_cache.get_table( obj.t_id ); + const auto& table_obj = db_iter_store.get_table( obj.t_id ); EOS_ASSERT( table_obj.code == receiver, table_access_violation, "db access violation" ); // require_write_lock( table_obj.scope ); @@ -707,14 +895,25 @@ void apply_context::db_update_i64( int iterator, account_name payer, const char* if( payer == account_name() ) payer = obj.payer; + std::string event_id; + if (control.get_deep_mind_logger() != nullptr) { + event_id = db_context::table_event(table_obj.code, table_obj.scope, table_obj.table, name(obj.primary_key)); + } + if( account_name(obj.payer) != payer ) { // refund the existing payer - update_db_usage( obj.payer, -(old_size) ); + update_db_usage( obj.payer, -(old_size), db_context::row_update_rem_trace(get_action_id(), std::string(event_id)) ); // charge the new payer - update_db_usage( payer, (new_size)); + update_db_usage( payer, (new_size), db_context::row_update_add_trace(get_action_id(), std::move(event_id)) ); } else if(old_size != new_size) { // charge/refund the existing payer the difference - update_db_usage( obj.payer, new_size - old_size); + update_db_usage( obj.payer, new_size - old_size, db_context::row_update_trace(get_action_id(), std::move(event_id)) ); + } + + if (auto dm_logger = control.get_deep_mind_logger()) { + db_context::log_row_update(*dm_logger, get_action_id(), table_obj.code, table_obj.scope, table_obj.table, + obj.payer, payer, name(obj.primary_key), obj.value.data(), obj.value.size(), + buffer, buffer_size); } db.modify( obj, [&]( auto& o ) { @@ -723,15 +922,24 @@ void apply_context::db_update_i64( int iterator, account_name payer, const char* }); } -void apply_context::db_remove_i64( int iterator ) { - const key_value_object& obj = keyval_cache.get( iterator ); +void apply_context::db_remove_i64_chainbase( int iterator ) { + const key_value_object& obj = db_iter_store.get( iterator ); - const auto& table_obj = keyval_cache.get_table( obj.t_id ); + const auto& table_obj = db_iter_store.get_table( obj.t_id ); EOS_ASSERT( table_obj.code == receiver, table_access_violation, "db access violation" ); // require_write_lock( table_obj.scope ); - update_db_usage( obj.payer, -(obj.value.size() + config::billable_size_v) ); + std::string event_id; + if (control.get_deep_mind_logger() != nullptr) { + event_id = db_context::table_event(table_obj.code, table_obj.scope, table_obj.table, name(obj.primary_key)); + } + + update_db_usage( obj.payer, -(obj.value.size() + config::billable_size_v), db_context::row_rem_trace(get_action_id(), std::move(event_id)) ); + + if (auto dm_logger = control.get_deep_mind_logger()) { + db_context::log_row_remove(*dm_logger, get_action_id(), table_obj.code, table_obj.scope, table_obj.table, obj.payer, name(obj.primary_key), obj.value.data(), obj.value.size()); + } db.modify( table_obj, [&]( auto& t ) { --t.count; @@ -742,11 +950,11 @@ void apply_context::db_remove_i64( int iterator ) { remove_table(table_obj); } - keyval_cache.remove( iterator ); + db_iter_store.remove( iterator ); } -int apply_context::db_get_i64( int iterator, char* buffer, size_t buffer_size ) { - const key_value_object& obj = keyval_cache.get( iterator ); +int apply_context::db_get_i64_chainbase( int iterator, char* buffer, size_t buffer_size ) { + const key_value_object& obj = db_iter_store.get( iterator ); auto s = obj.value.size(); if( buffer_size == 0 ) return s; @@ -757,27 +965,27 @@ int apply_context::db_get_i64( int iterator, char* buffer, size_t buffer_size ) return copy_size; } -int apply_context::db_next_i64( int iterator, uint64_t& primary ) { +int apply_context::db_next_i64_chainbase( int iterator, uint64_t& primary ) { if( iterator < -1 ) return -1; // cannot increment past end iterator of table - const auto& obj = keyval_cache.get( iterator ); // Check for iterator != -1 happens in this call + const auto& obj = db_iter_store.get( iterator ); // Check for iterator != -1 happens in this call const auto& idx = db.get_index(); auto itr = idx.iterator_to( obj ); ++itr; - if( itr == idx.end() || itr->t_id != obj.t_id ) return keyval_cache.get_end_iterator_by_table_id(obj.t_id); + if( itr == idx.end() || itr->t_id != obj.t_id ) return db_iter_store.get_end_iterator_by_table_id(obj.t_id); primary = itr->primary_key; - return keyval_cache.add( *itr ); + return db_iter_store.add( *itr ); } -int apply_context::db_previous_i64( int iterator, uint64_t& primary ) { +int apply_context::db_previous_i64_chainbase( int iterator, uint64_t& primary ) { const auto& idx = db.get_index(); if( iterator < -1 ) // is end iterator { - auto tab = keyval_cache.find_table_by_end_iterator(iterator); + auto tab = db_iter_store.find_table_by_end_iterator(iterator); EOS_ASSERT( tab, invalid_table_iterator, "not a valid end iterator" ); auto itr = idx.upper_bound(tab->id); @@ -788,10 +996,10 @@ int apply_context::db_previous_i64( int iterator, uint64_t& primary ) { if( itr->t_id != tab->id ) return -1; // Empty table primary = itr->primary_key; - return keyval_cache.add(*itr); + return db_iter_store.add(*itr); } - const auto& obj = keyval_cache.get(iterator); // Check for iterator != -1 happens in this call + const auto& obj = db_iter_store.get(iterator); // Check for iterator != -1 happens in this call auto itr = idx.iterator_to(obj); if( itr == idx.begin() ) return -1; // cannot decrement past beginning iterator of table @@ -801,62 +1009,149 @@ int apply_context::db_previous_i64( int iterator, uint64_t& primary ) { if( itr->t_id != obj.t_id ) return -1; // cannot decrement past beginning iterator of table primary = itr->primary_key; - return keyval_cache.add(*itr); + return db_iter_store.add(*itr); } -int apply_context::db_find_i64( name code, name scope, name table, uint64_t id ) { +int apply_context::db_find_i64_chainbase( name code, name scope, name table, uint64_t id ) { //require_read_lock( code, scope ); // redundant? const auto* tab = find_table( code, scope, table ); if( !tab ) return -1; - auto table_end_itr = keyval_cache.cache_table( *tab ); + auto table_end_itr = db_iter_store.cache_table( *tab ); const key_value_object* obj = db.find( boost::make_tuple( tab->id, id ) ); if( !obj ) return table_end_itr; - return keyval_cache.add( *obj ); + return db_iter_store.add( *obj ); } -int apply_context::db_lowerbound_i64( name code, name scope, name table, uint64_t id ) { +int apply_context::db_lowerbound_i64_chainbase( name code, name scope, name table, uint64_t id ) { //require_read_lock( code, scope ); // redundant? const auto* tab = find_table( code, scope, table ); if( !tab ) return -1; - auto table_end_itr = keyval_cache.cache_table( *tab ); + auto table_end_itr = db_iter_store.cache_table( *tab ); const auto& idx = db.get_index(); auto itr = idx.lower_bound( boost::make_tuple( tab->id, id ) ); if( itr == idx.end() ) return table_end_itr; if( itr->t_id != tab->id ) return table_end_itr; - return keyval_cache.add( *itr ); + return db_iter_store.add( *itr ); } -int apply_context::db_upperbound_i64( name code, name scope, name table, uint64_t id ) { +int apply_context::db_upperbound_i64_chainbase( name code, name scope, name table, uint64_t id ) { //require_read_lock( code, scope ); // redundant? const auto* tab = find_table( code, scope, table ); if( !tab ) return -1; - auto table_end_itr = keyval_cache.cache_table( *tab ); + auto table_end_itr = db_iter_store.cache_table( *tab ); const auto& idx = db.get_index(); auto itr = idx.upper_bound( boost::make_tuple( tab->id, id ) ); if( itr == idx.end() ) return table_end_itr; if( itr->t_id != tab->id ) return table_end_itr; - return keyval_cache.add( *itr ); + return db_iter_store.add( *itr ); } -int apply_context::db_end_i64( name code, name scope, name table ) { +int apply_context::db_end_i64_chainbase( name code, name scope, name table ) { //require_read_lock( code, scope ); // redundant? const auto* tab = find_table( code, scope, table ); if( !tab ) return -1; - return keyval_cache.cache_table( *tab ); + return db_iter_store.cache_table( *tab ); +} + +int64_t apply_context::kv_erase(uint64_t contract, const char* key, uint32_t key_size) { + return kv_get_backing_store().kv_erase(contract, key, key_size); +} + +int64_t apply_context::kv_set(uint64_t contract, const char* key, uint32_t key_size, const char* value, uint32_t value_size, account_name payer) { + return kv_get_backing_store().kv_set(contract, key, key_size, value, value_size, payer); +} + +bool apply_context::kv_get(uint64_t contract, const char* key, uint32_t key_size, uint32_t& value_size) { + return kv_get_backing_store().kv_get(contract, key, key_size, value_size); +} + +uint32_t apply_context::kv_get_data(uint32_t offset, char* data, uint32_t data_size) { + return kv_get_backing_store().kv_get_data(offset, data, data_size); +} + +uint32_t apply_context::kv_it_create(uint64_t contract, const char* prefix, uint32_t size) { + uint32_t itr; + if (!kv_destroyed_iterators.empty()) { + itr = kv_destroyed_iterators.back(); + kv_destroyed_iterators.pop_back(); + } else { + // Sanity check in case the per-database limits are set poorly + EOS_ASSERT(kv_iterators.size() <= 0xFFFFFFFFu, kv_bad_iter, "Too many iterators"); + itr = kv_iterators.size(); + kv_iterators.emplace_back(); + } + kv_iterators[itr] = kv_get_backing_store().kv_it_create(contract, prefix, size); + return itr; +} + +void apply_context::kv_it_destroy(uint32_t itr) { + kv_check_iterator(itr); + kv_destroyed_iterators.push_back(itr); + kv_iterators[itr].reset(); +} + +int32_t apply_context::kv_it_status(uint32_t itr) { + kv_check_iterator(itr); + return static_cast(kv_iterators[itr]->kv_it_status()); +} + +int32_t apply_context::kv_it_compare(uint32_t itr_a, uint32_t itr_b) { + kv_check_iterator(itr_a); + kv_check_iterator(itr_b); + return kv_iterators[itr_a]->kv_it_compare(*kv_iterators[itr_b]); +} + +int32_t apply_context::kv_it_key_compare(uint32_t itr, const char* key, uint32_t size) { + kv_check_iterator(itr); + return kv_iterators[itr]->kv_it_key_compare(key, size); +} + +int32_t apply_context::kv_it_move_to_end(uint32_t itr) { + kv_check_iterator(itr); + return static_cast(kv_iterators[itr]->kv_it_move_to_end()); +} + +int32_t apply_context::kv_it_next(uint32_t itr, uint32_t* found_key_size, uint32_t* found_value_size) { + kv_check_iterator(itr); + return static_cast(kv_iterators[itr]->kv_it_next(found_key_size, found_value_size)); +} + +int32_t apply_context::kv_it_prev(uint32_t itr, uint32_t* found_key_size, uint32_t* found_value_size) { + kv_check_iterator(itr); + return static_cast(kv_iterators[itr]->kv_it_prev(found_key_size, found_value_size)); +} + +int32_t apply_context::kv_it_lower_bound(uint32_t itr, const char* key, uint32_t size, uint32_t* found_key_size, uint32_t* found_value_size) { + kv_check_iterator(itr); + return static_cast(kv_iterators[itr]->kv_it_lower_bound(key, size, found_key_size, found_value_size)); +} + +int32_t apply_context::kv_it_key(uint32_t itr, uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) { + kv_check_iterator(itr); + return static_cast(kv_iterators[itr]->kv_it_key(offset, dest, size, actual_size)); +} + +int32_t apply_context::kv_it_value(uint32_t itr, uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) { + kv_check_iterator(itr); + return static_cast(kv_iterators[itr]->kv_it_value(offset, dest, size, actual_size)); +} + +void apply_context::kv_check_iterator(uint32_t itr) { + EOS_ASSERT(itr < kv_iterators.size() && kv_iterators[itr], kv_bad_iter, "Bad key-value iterator"); } uint64_t apply_context::next_global_sequence() { @@ -881,8 +1176,8 @@ uint64_t apply_context::next_auth_sequence( account_name actor ) { return amo.auth_sequence; } -void apply_context::add_ram_usage( account_name account, int64_t ram_delta ) { - trx_context.add_ram_usage( account, ram_delta ); +void apply_context::add_ram_usage( account_name account, int64_t ram_delta, const storage_usage_trace& trace ) { + trx_context.add_ram_usage( account, ram_delta, trace ); auto p = _account_ram_deltas.emplace( account, ram_delta ); if( !p.second ) { @@ -899,4 +1194,17 @@ action_name apply_context::get_sender() const { return action_name(); } +uint32_t apply_context::get_action_id() const { + return trx_context.action_id.current(); +} + +void apply_context::increment_action_id() { + trx_context.action_id.increment(); +} + +db_context& apply_context::db_get_context() { + EOS_ASSERT( _db_context, action_validate_exception, + "context-free actions cannot access state" ); + return *_db_context; +} } } /// eosio::chain diff --git a/libraries/chain/authority.cpp b/libraries/chain/authority.cpp new file mode 100644 index 00000000000..d36640a476c --- /dev/null +++ b/libraries/chain/authority.cpp @@ -0,0 +1,7 @@ +#include + +namespace fc { + void to_variant(const eosio::chain::shared_public_key& var, fc::variant& vo) { + vo = var.to_string(); + } +} // namespace fc diff --git a/libraries/chain/authorization_manager.cpp b/libraries/chain/authorization_manager.cpp index 9e76ff63bf3..1427b3c3bff 100644 --- a/libraries/chain/authorization_manager.cpp +++ b/libraries/chain/authorization_manager.cpp @@ -133,11 +133,12 @@ namespace eosio { namespace chain { permission_name name, permission_id_type parent, const authority& auth, + uint32_t action_id, time_point initial_creation_time ) { for(const key_weight& k: auth.keys) - EOS_ASSERT(k.key.which() < _db.get().num_supported_key_types, unactivated_key_type, + EOS_ASSERT(static_cast(k.key.which()) < _db.get().num_supported_key_types, unactivated_key_type, "Unactivated key type used when creating permission"); auto creation_time = initial_creation_time; @@ -156,6 +157,14 @@ namespace eosio { namespace chain { p.name = name; p.last_updated = creation_time; p.auth = auth; + + if (auto dm_logger = _control.get_deep_mind_logger()) { + fc_dlog(*dm_logger, "PERM_OP INS ${action_id} ${permission_id} ${data}", + ("action_id", action_id) + ("permission_id", p.id) + ("data", p) + ); + } }); return perm; } @@ -164,11 +173,12 @@ namespace eosio { namespace chain { permission_name name, permission_id_type parent, authority&& auth, + uint32_t action_id, time_point initial_creation_time ) { for(const key_weight& k: auth.keys) - EOS_ASSERT(k.key.which() < _db.get().num_supported_key_types, unactivated_key_type, + EOS_ASSERT(static_cast(k.key.which()) < _db.get().num_supported_key_types, unactivated_key_type, "Unactivated key type used when creating permission"); auto creation_time = initial_creation_time; @@ -187,28 +197,63 @@ namespace eosio { namespace chain { p.name = name; p.last_updated = creation_time; p.auth = std::move(auth); + + if (auto dm_logger = _control.get_deep_mind_logger()) { + fc_dlog(*dm_logger, "PERM_OP INS ${action_id} ${permission_id} ${data}", + ("action_id", action_id) + ("permission_id", p.id) + ("data", p) + ); + } }); return perm; } - void authorization_manager::modify_permission( const permission_object& permission, const authority& auth ) { + void authorization_manager::modify_permission( const permission_object& permission, const authority& auth, uint32_t action_id ) { for(const key_weight& k: auth.keys) - EOS_ASSERT(k.key.which() < _db.get().num_supported_key_types, unactivated_key_type, + EOS_ASSERT(static_cast(k.key.which()) < _db.get().num_supported_key_types, unactivated_key_type, "Unactivated key type used when modifying permission"); _db.modify( permission, [&](permission_object& po) { + auto dm_logger = _control.get_deep_mind_logger(); + + fc::variant old_permission; + if (dm_logger) { + old_permission = po; + } + po.auth = auth; po.last_updated = _control.pending_block_time(); + + if (auto dm_logger = _control.get_deep_mind_logger()) { + fc_dlog(*dm_logger, "PERM_OP UPD ${action_id} ${permission_id} ${data}", + ("action_id", action_id) + ("permission_id", po.id) + ("data", fc::mutable_variant_object() + ("old", old_permission) + ("new", po) + ) + ); + } }); } - void authorization_manager::remove_permission( const permission_object& permission ) { + void authorization_manager::remove_permission( const permission_object& permission, uint32_t action_id) { const auto& index = _db.template get_index(); auto range = index.equal_range(permission.id); EOS_ASSERT( range.first == range.second, action_validate_exception, "Cannot remove a permission which has children. Remove the children first."); _db.get_mutable_index().remove_object( permission.usage_id._id ); + + if (auto dm_logger = _control.get_deep_mind_logger()) { + fc_dlog(*dm_logger, "PERM_OP REM ${action_id} ${permission_id} ${data}", + ("action_id", action_id) + ("permission_id", permission.id) + ("data", permission) + ); + } + _db.remove( permission ); } @@ -235,10 +280,10 @@ namespace eosio { namespace chain { return _db.get( boost::make_tuple(level.actor,level.permission) ); } EOS_RETHROW_EXCEPTIONS( chain::permission_query_exception, "Failed to retrieve permission: ${level}", ("level", level) ) } - optional authorization_manager::lookup_linked_permission( account_name authorizer_account, - account_name scope, - action_name act_name - )const + std::optional authorization_manager::lookup_linked_permission( account_name authorizer_account, + account_name scope, + action_name act_name + )const { try { // First look up a specific link for this message act_name @@ -254,16 +299,14 @@ namespace eosio { namespace chain { if (link != nullptr) { return link->required_permission; } - return optional(); - - // return optional(); + return std::optional(); } FC_CAPTURE_AND_RETHROW((authorizer_account)(scope)(act_name)) } - optional authorization_manager::lookup_minimum_permission( account_name authorizer_account, - account_name scope, - action_name act_name - )const + std::optional authorization_manager::lookup_minimum_permission( account_name authorizer_account, + account_name scope, + action_name act_name + )const { // Special case native actions cannot be linked to a minimum permission, so there is no need to check. if( scope == config::system_account_name ) { @@ -277,12 +320,12 @@ namespace eosio { namespace chain { } try { - optional linked_permission = lookup_linked_permission(authorizer_account, scope, act_name); + std::optional linked_permission = lookup_linked_permission(authorizer_account, scope, act_name); if( !linked_permission ) return config::active_name; if( *linked_permission == config::eosio_any_name ) - return optional(); + return std::optional(); return linked_permission; } FC_CAPTURE_AND_RETHROW((authorizer_account)(scope)(act_name)) @@ -340,7 +383,7 @@ namespace eosio { namespace chain { "the owner of the linked permission needs to be the actor of the declared authorization" ); if( link.code == config::system_account_name - || !_control.is_builtin_activated( builtin_protocol_feature_t::fix_linkauth_restriction ) ) + || !_control.is_builtin_activated( builtin_protocol_feature_t::fix_linkauth_restriction ) ) { EOS_ASSERT( link.type != updateauth::get_name(), action_validate_exception, "Cannot link eosio::updateauth to a minimum permission" ); @@ -377,7 +420,7 @@ namespace eosio { namespace chain { "the owner of the linked permission needs to be the actor of the declared authorization" ); const auto unlinked_permission_name = lookup_linked_permission(unlink.account, unlink.code, unlink.type); - EOS_ASSERT( unlinked_permission_name.valid(), transaction_exception, + EOS_ASSERT( unlinked_permission_name, transaction_exception, "cannot unlink non-existent permission link of account '${account}' for actions matching '${code}::${action}'", ("account", unlink.account)("code", unlink.code)("action", unlink.type) ); diff --git a/libraries/chain/backing_store/db_context.cpp b/libraries/chain/backing_store/db_context.cpp new file mode 100644 index 00000000000..f5a0c603525 --- /dev/null +++ b/libraries/chain/backing_store/db_context.cpp @@ -0,0 +1,134 @@ +#include +#include +#include + +namespace eosio { namespace chain { namespace backing_store { + +std::string db_context::table_event(name code, name scope, name table) { + return STORAGE_EVENT_ID("${code}:${scope}:${table}", + ("code", code) + ("scope", scope) + ("table", table) + ); +} + +std::string db_context::table_event(name code, name scope, name table, name qualifier) { + return STORAGE_EVENT_ID("${code}:${scope}:${table}:${qualifier}", + ("code", code) + ("scope", scope) + ("table", table) + ("qualifier", qualifier) + ); +} + +void db_context::log_insert_table(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, account_name payer) { + fc_dlog(deep_mind_logger, "TBL_OP INS ${action_id} ${code} ${scope} ${table} ${payer}", + ("action_id", action_id) + ("code", code) + ("scope", scope) + ("table", table) + ("payer", payer) + ); +} + +void db_context::log_remove_table(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, account_name payer) { + fc_dlog(deep_mind_logger, "TBL_OP REM ${action_id} ${code} ${scope} ${table} ${payer}", + ("action_id", action_id) + ("code", code) + ("scope", scope) + ("table", table) + ("payer", payer) + ); +} + +void db_context::log_row_insert(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, + account_name payer, account_name primkey, const char* buffer, size_t buffer_size) { + fc_dlog(deep_mind_logger, "DB_OP INS ${action_id} ${payer} ${table_code} ${scope} ${table_name} ${primkey} ${ndata}", + ("action_id", action_id) + ("payer", payer) + ("table_code", code) + ("scope", scope) + ("table_name", table) + ("primkey", primkey) + ("ndata", fc::to_hex(buffer, buffer_size)) + ); +} + +void db_context::log_row_update(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, + account_name old_payer, account_name new_payer, account_name primkey, + const char* old_buffer, size_t old_buffer_size, const char* new_buffer, size_t new_buffer_size) { + fc_dlog(deep_mind_logger, "DB_OP UPD ${action_id} ${opayer}:${npayer} ${table_code} ${scope} ${table_name} ${primkey} ${odata}:${ndata}", + ("action_id", action_id) + ("opayer", old_payer) + ("npayer", new_payer) + ("table_code", code) + ("scope", scope) + ("table_name", table) + ("primkey", primkey) + ("odata", to_hex(old_buffer, old_buffer_size)) + ("ndata", to_hex(new_buffer, new_buffer_size)) + ); +} + +void db_context::log_row_remove(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, + account_name payer, account_name primkey, const char* buffer, size_t buffer_size) { + fc_dlog(deep_mind_logger, "DB_OP REM ${action_id} ${payer} ${table_code} ${scope} ${table_name} ${primkey} ${odata}", + ("action_id", action_id) + ("payer", payer) + ("table_code", code) + ("scope", scope) + ("table_name", table) + ("primkey", primkey) + ("odata", fc::to_hex(buffer, buffer_size)) + ); +} + +storage_usage_trace db_context::add_table_trace(uint32_t action_id, std::string&& event_id) { + return storage_usage_trace(action_id, std::move(event_id), "table", "add", "create_table"); +} + +storage_usage_trace db_context::rem_table_trace(uint32_t action_id, std::string&& event_id) { + return storage_usage_trace(action_id, std::move(event_id), "table", "remove", "remove_table"); +} + +storage_usage_trace db_context::row_add_trace(uint32_t action_id, std::string&& event_id) { + return storage_usage_trace(action_id, std::move(event_id), "table_row", "add", "primary_index_add"); +} + +storage_usage_trace db_context::row_update_trace(uint32_t action_id, std::string&& event_id) { + return storage_usage_trace(action_id, std::move(event_id), "table_row", "update", "primary_index_update"); +} + +storage_usage_trace db_context::row_update_add_trace(uint32_t action_id, std::string&& event_id) { + return storage_usage_trace(action_id, std::move(event_id), "table_row", "add", "primary_index_update_add_new_payer"); +} + +storage_usage_trace db_context::row_update_rem_trace(uint32_t action_id, std::string&& event_id) { + return storage_usage_trace(action_id, std::move(event_id), "table_row", "remove", "primary_index_update_remove_old_payer"); +} + +storage_usage_trace db_context::row_rem_trace(uint32_t action_id, std::string&& event_id) { + return storage_usage_trace(action_id, std::move(event_id), "table_row", "remove", "primary_index_remove"); +} + +storage_usage_trace db_context::secondary_add_trace(uint32_t action_id, std::string&& event_id) { + return storage_usage_trace(action_id, std::move(event_id), "secondary_index", "add", "secondary_index_add"); +} + +storage_usage_trace db_context::secondary_rem_trace(uint32_t action_id, std::string&& event_id) { + return storage_usage_trace(action_id, std::move(event_id), "secondary_index", "remove", "secondary_index_remove"); +} + +storage_usage_trace db_context::secondary_update_add_trace(uint32_t action_id, std::string&& event_id) { + return storage_usage_trace(action_id, std::move(event_id), "secondary_index", "add", "secondary_index_update_add_new_payer"); +} + +storage_usage_trace db_context::secondary_update_rem_trace(uint32_t action_id, std::string&& event_id) { + return storage_usage_trace(action_id, std::move(event_id), "secondary_index", "remove", "secondary_index_update_remove_old_payer"); +} + +void db_context::update_db_usage( const account_name& payer, int64_t delta, const storage_usage_trace& trace ) { + context.update_db_usage(payer, delta, trace); +} + +}}} // namespace eosio::chain::backing_store diff --git a/libraries/chain/backing_store/db_context_chainbase.cpp b/libraries/chain/backing_store/db_context_chainbase.cpp new file mode 100644 index 00000000000..b5a15aca8e1 --- /dev/null +++ b/libraries/chain/backing_store/db_context_chainbase.cpp @@ -0,0 +1,297 @@ +#include +#include +#include + +namespace eosio { namespace chain { namespace backing_store { + + struct db_context_chainbase : db_context { + db_context_chainbase(apply_context& context, name receiver) + : db_context( context, receiver ) {} + + int32_t db_store_i64(uint64_t scope, uint64_t table, account_name payer, uint64_t id, const char* buffer , size_t buffer_size) override { + return context.db_store_i64_chainbase(name(scope), name(table), payer, id, buffer, buffer_size); + } + + void db_update_i64(int32_t itr, account_name payer, const char* buffer , size_t buffer_size) override { + context.db_update_i64_chainbase(itr, payer, buffer, buffer_size); + } + + void db_remove_i64(int32_t itr) override { + context.db_remove_i64_chainbase(itr); + } + + int32_t db_get_i64(int32_t itr, char* buffer , size_t buffer_size) override { + return context.db_get_i64_chainbase(itr, buffer, buffer_size); + } + + int32_t db_next_i64(int32_t itr, uint64_t& primary) override { + return context.db_next_i64_chainbase(itr, primary); + } + + int32_t db_previous_i64(int32_t itr, uint64_t& primary) override { + return context.db_previous_i64_chainbase(itr, primary); + } + + int32_t db_find_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) override { + return context.db_find_i64_chainbase(name(code), name(scope), name(table), id); + } + + int32_t db_lowerbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) override { + return context.db_lowerbound_i64_chainbase(name(code), name(scope), name(table), id); + } + + int32_t db_upperbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) override { + return context.db_upperbound_i64_chainbase(name(code), name(scope), name(table), id); + } + + int32_t db_end_i64(uint64_t code, uint64_t scope, uint64_t table) override { + return context.db_end_i64_chainbase(name(code), name(scope), name(table)); + } + + /** + * interface for uint64_t secondary + */ + int32_t db_idx64_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, + const uint64_t& secondary) override { + return context.idx64.store( scope, table, payer, id, secondary ); + } + + void db_idx64_update(int32_t iterator, account_name payer, const uint64_t& secondary) override { + context.idx64.update( iterator, payer, secondary ); + } + + void db_idx64_remove(int32_t iterator) override { + return context.idx64.remove(iterator); + } + + int32_t db_idx64_find_secondary(uint64_t code, uint64_t scope, uint64_t table, const uint64_t& secondary, + uint64_t& primary) override { + return context.idx64.find_secondary(code, scope, table, secondary, primary); + } + + int32_t db_idx64_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, + uint64_t primary) override { + return context.idx64.find_primary(code, scope, table, secondary, primary); + } + + int32_t db_idx64_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, + uint64_t& primary) override { + return context.idx64.lowerbound_secondary(code, scope, table, secondary, primary); + } + + int32_t db_idx64_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, + uint64_t& primary) override { + return context.idx64.upperbound_secondary(code, scope, table, secondary, primary); + } + + int32_t db_idx64_end(uint64_t code, uint64_t scope, uint64_t table) override { + return context.idx64.end_secondary(code, scope, table); + } + + int32_t db_idx64_next(int32_t iterator, uint64_t& primary) override { + return context.idx64.next_secondary(iterator, primary); + } + + int32_t db_idx64_previous(int32_t iterator, uint64_t& primary) override { + return context.idx64.previous_secondary(iterator, primary); + } + + /** + * interface for uint128_t secondary + */ + int32_t db_idx128_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, + const uint128_t& secondary) override { + return context.idx128.store(scope, table, payer, id, secondary); + } + + void db_idx128_update(int32_t iterator, account_name payer, const uint128_t& secondary) override { + return context.idx128.update(iterator, payer, secondary); + } + + void db_idx128_remove(int32_t iterator) override { + context.idx128.remove(iterator); + } + + int32_t db_idx128_find_secondary(uint64_t code, uint64_t scope, uint64_t table, const uint128_t& secondary, + uint64_t& primary) override { + return context.idx128.find_secondary(code, scope, table, secondary, primary); + } + + int32_t db_idx128_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, + uint64_t primary) override { + return context.idx128.find_primary(code, scope, table, secondary, primary); + } + + int32_t db_idx128_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, + uint64_t& primary) override { + return context.idx128.lowerbound_secondary(code, scope, table, secondary, primary); + } + + int32_t db_idx128_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, + uint64_t& primary) override { + return context.idx128.upperbound_secondary(code, scope, table, secondary, primary); + } + + int32_t db_idx128_end(uint64_t code, uint64_t scope, uint64_t table) override { + return context.idx128.end_secondary(code, scope, table); + } + + int32_t db_idx128_next(int32_t iterator, uint64_t& primary) override { + return context.idx128.next_secondary(iterator, primary); + } + + int32_t db_idx128_previous(int32_t iterator, uint64_t& primary) override { + return context.idx128.previous_secondary(iterator, primary); + } + + /** + * interface for 256-bit interger secondary + */ + int32_t db_idx256_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, + const uint128_t* data) override { + return context.idx256.store(scope, table, payer, id, data); + } + + void db_idx256_update(int32_t iterator, account_name payer, const uint128_t* data) override { + context.idx256.update(iterator, payer, data); + } + + void db_idx256_remove(int32_t iterator) override { + context.idx256.remove(iterator); + } + + int32_t db_idx256_find_secondary(uint64_t code, uint64_t scope, uint64_t table, const uint128_t* data, + uint64_t& primary) override { + return context.idx256.find_secondary(code, scope, table, data, primary); + } + + int32_t db_idx256_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, + uint64_t primary) override { + return context.idx256.find_primary(code, scope, table, data, primary); + } + + int32_t db_idx256_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, + uint64_t& primary) override { + return context.idx256.lowerbound_secondary(code, scope, table, data, primary); + } + + int32_t db_idx256_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, + uint64_t& primary) override { + return context.idx256.upperbound_secondary(code, scope, table, data, primary); + } + + int32_t db_idx256_end(uint64_t code, uint64_t scope, uint64_t table) override { + return context.idx256.end_secondary(code, scope, table); + } + + int32_t db_idx256_next(int32_t iterator, uint64_t& primary) override { + return context.idx256.next_secondary(iterator, primary); + } + + int32_t db_idx256_previous(int32_t iterator, uint64_t& primary) override { + return context.idx256.previous_secondary(iterator, primary); + } + + /** + * interface for double secondary + */ + int32_t db_idx_double_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, + const float64_t& secondary) override { + return context.idx_double.store(scope, table, payer, id, secondary); + } + + void db_idx_double_update(int32_t iterator, account_name payer, const float64_t& secondary) override { + context.idx_double.update(iterator, payer, secondary); + } + + void db_idx_double_remove(int32_t iterator) override { + context.idx_double.remove(iterator); + } + + int32_t db_idx_double_find_secondary(uint64_t code, uint64_t scope, uint64_t table, + const float64_t& secondary, uint64_t& primary) override { + return context.idx_double.find_secondary(code, scope, table, secondary, primary); + } + + int32_t db_idx_double_find_primary(uint64_t code, uint64_t scope, uint64_t table, float64_t& secondary, + uint64_t primary) override { + return context.idx_double.find_primary(code, scope, table, secondary, primary); + } + + int32_t db_idx_double_lowerbound(uint64_t code, uint64_t scope, uint64_t table, float64_t& secondary, + uint64_t& primary) override { + return context.idx_double.lowerbound_secondary(code, scope, table, secondary, primary); + } + + int32_t db_idx_double_upperbound(uint64_t code, uint64_t scope, uint64_t table, float64_t& secondary, + uint64_t& primary) override { + return context.idx_double.upperbound_secondary(code, scope, table, secondary, primary); + } + + int32_t db_idx_double_end(uint64_t code, uint64_t scope, uint64_t table) override { + return context.idx_double.end_secondary(code, scope, table); + } + + int32_t db_idx_double_next(int32_t iterator, uint64_t& primary) override { + return context.idx_double.next_secondary(iterator, primary); + } + + int32_t db_idx_double_previous(int32_t iterator, uint64_t& primary) override { + return context.idx_double.previous_secondary(iterator, primary); + } + + /** + * interface for long double secondary + */ + int32_t db_idx_long_double_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, + const float128_t& secondary) override { + return context.idx_long_double.store(scope, table, payer, id, secondary); + } + + void db_idx_long_double_update(int32_t iterator, account_name payer, const float128_t& secondary) override { + context.idx_long_double.update(iterator, payer, secondary); + } + + void db_idx_long_double_remove(int32_t iterator) override { + context.idx_long_double.remove(iterator); + } + + int32_t db_idx_long_double_find_secondary(uint64_t code, uint64_t scope, uint64_t table, + const float128_t& secondary, uint64_t& primary) override { + return context.idx_long_double.find_secondary(code, scope, table, secondary, primary); + } + + int32_t db_idx_long_double_find_primary(uint64_t code, uint64_t scope, uint64_t table, + float128_t& secondary, uint64_t primary) override { + return context.idx_long_double.find_primary(code, scope, table, secondary, primary); + } + + int32_t db_idx_long_double_lowerbound(uint64_t code, uint64_t scope, uint64_t table, float128_t& secondary, + uint64_t& primary) override { + return context.idx_long_double.lowerbound_secondary(code, scope, table, secondary, primary); + } + + int32_t db_idx_long_double_upperbound(uint64_t code, uint64_t scope, uint64_t table, float128_t& secondary, + uint64_t& primary) override { + return context.idx_long_double.upperbound_secondary(code, scope, table, secondary, primary); + } + + int32_t db_idx_long_double_end(uint64_t code, uint64_t scope, uint64_t table) override { + return context.idx_long_double.end_secondary(code, scope, table); + } + + int32_t db_idx_long_double_next(int32_t iterator, uint64_t& primary) override { + return context.idx_long_double.next_secondary(iterator, primary); + } + + int32_t db_idx_long_double_previous(int32_t iterator, uint64_t& primary) override { + return context.idx_long_double.previous_secondary(iterator, primary); + } + }; // db_context_chainbase + + std::unique_ptr create_db_chainbase_context(apply_context& context, name receiver) + { + return std::make_unique(context, receiver); + } + +}}} // namespace eosio::chain::backing_store diff --git a/libraries/chain/backing_store/db_context_rocksdb.cpp b/libraries/chain/backing_store/db_context_rocksdb.cpp new file mode 100644 index 00000000000..e6c04369ee3 --- /dev/null +++ b/libraries/chain/backing_store/db_context_rocksdb.cpp @@ -0,0 +1,764 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace eosio { namespace chain { namespace backing_store { + + class db_context_rocksdb : public db_context { + public: + using prim_key_iter_type = secondary_key; + using session_type = eosio::session::session>; + using session_variant_type = eosio::session::session_variant; + using shared_bytes = eosio::session::shared_bytes; + + db_context_rocksdb(apply_context& context, name receiver, session_variant_type session); + + ~db_context_rocksdb() override; + + int32_t db_store_i64(uint64_t scope, uint64_t table, account_name payer, uint64_t id, const char* value , size_t value_size) override; + void db_update_i64(int32_t itr, account_name payer, const char* value , size_t value_size) override; + void db_remove_i64(int32_t itr) override; + int32_t db_get_i64(int32_t itr, char* value , size_t value_size) override; + int32_t db_next_i64(int32_t itr, uint64_t& primary) override; + int32_t db_previous_i64(int32_t itr, uint64_t& primary) override; + int32_t db_find_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) override; + int32_t db_lowerbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) override; + int32_t db_upperbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) override; + int32_t db_end_i64(uint64_t code, uint64_t scope, uint64_t table) override; + + /** + * interface for uint64_t secondary + */ + int32_t db_idx64_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, + const uint64_t& secondary) override; + void db_idx64_update(int32_t iterator, account_name payer, const uint64_t& secondary) override; + void db_idx64_remove(int32_t iterator) override; + int32_t db_idx64_find_secondary(uint64_t code, uint64_t scope, uint64_t table, const uint64_t& secondary, + uint64_t& primary) override; + int32_t db_idx64_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, + uint64_t primary) override; + int32_t db_idx64_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, + uint64_t& primary) override; + int32_t db_idx64_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, + uint64_t& primary) override; + int32_t db_idx64_end(uint64_t code, uint64_t scope, uint64_t table) override; + int32_t db_idx64_next(int32_t iterator, uint64_t& primary) override; + int32_t db_idx64_previous(int32_t iterator, uint64_t& primary) override; + + /** + * interface for uint128_t secondary + */ + int32_t db_idx128_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, + const uint128_t& secondary) override; + void db_idx128_update(int32_t iterator, account_name payer, const uint128_t& secondary) override; + void db_idx128_remove(int32_t iterator) override; + int32_t db_idx128_find_secondary(uint64_t code, uint64_t scope, uint64_t table, const uint128_t& secondary, + uint64_t& primary) override; + int32_t db_idx128_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, + uint64_t primary) override; + int32_t db_idx128_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, + uint64_t& primary) override; + int32_t db_idx128_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, + uint64_t& primary) override; + int32_t db_idx128_end(uint64_t code, uint64_t scope, uint64_t table) override; + int32_t db_idx128_next(int32_t iterator, uint64_t& primary) override; + int32_t db_idx128_previous(int32_t iterator, uint64_t& primary) override; + + /** + * interface for 256-bit interger secondary + */ + int32_t db_idx256_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, + const uint128_t* data) override; + void db_idx256_update(int32_t iterator, account_name payer, const uint128_t* data) override; + + void db_idx256_remove(int32_t iterator) override; + int32_t db_idx256_find_secondary(uint64_t code, uint64_t scope, uint64_t table, const uint128_t* data, + uint64_t& primary) override; + int32_t db_idx256_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, + uint64_t primary) override; + int32_t db_idx256_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, + uint64_t& primary) override; + int32_t db_idx256_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, + uint64_t& primary) override; + int32_t db_idx256_end(uint64_t code, uint64_t scope, uint64_t table) override; + int32_t db_idx256_next(int32_t iterator, uint64_t& primary) override; + int32_t db_idx256_previous(int32_t iterator, uint64_t& primary) override; + + /** + * interface for double secondary + */ + int32_t db_idx_double_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, + const float64_t& secondary) override; + void db_idx_double_update(int32_t iterator, account_name payer, const float64_t& secondary) override; + void db_idx_double_remove(int32_t iterator) override; + int32_t db_idx_double_find_secondary(uint64_t code, uint64_t scope, uint64_t table, + const float64_t& secondary, uint64_t& primary) override; + int32_t db_idx_double_find_primary(uint64_t code, uint64_t scope, uint64_t table, float64_t& secondary, + uint64_t primary) override; + int32_t db_idx_double_lowerbound(uint64_t code, uint64_t scope, uint64_t table, float64_t& secondary, + uint64_t& primary) override; + int32_t db_idx_double_upperbound(uint64_t code, uint64_t scope, uint64_t table, float64_t& secondary, + uint64_t& primary) override; + int32_t db_idx_double_end(uint64_t code, uint64_t scope, uint64_t table) override; + int32_t db_idx_double_next(int32_t iterator, uint64_t& primary) override; + int32_t db_idx_double_previous(int32_t iterator, uint64_t& primary) override; + /** + * interface for long double secondary + */ + int32_t db_idx_long_double_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, + const float128_t& secondary) override; + void db_idx_long_double_update(int32_t iterator, account_name payer, const float128_t& secondary) override; + void db_idx_long_double_remove(int32_t iterator) override; + int32_t db_idx_long_double_find_secondary(uint64_t code, uint64_t scope, uint64_t table, + const float128_t& secondary, uint64_t& primary) override; + int32_t db_idx_long_double_find_primary(uint64_t code, uint64_t scope, uint64_t table, + float128_t& secondary, uint64_t primary) override; + int32_t db_idx_long_double_lowerbound(uint64_t code, uint64_t scope, uint64_t table, float128_t& secondary, + uint64_t& primary) override; + int32_t db_idx_long_double_upperbound(uint64_t code, uint64_t scope, uint64_t table, float128_t& secondary, + uint64_t& primary) override; + int32_t db_idx_long_double_end(uint64_t code, uint64_t scope, uint64_t table) override; + int32_t db_idx_long_double_next(int32_t iterator, uint64_t& primary) override; + int32_t db_idx_long_double_previous(int32_t iterator, uint64_t& primary) override; + + // for primary keys in the iterator store, we don't care about secondary key + static prim_key_iter_type primary_key_iter(int table_ei, uint64_t key, account_name payer); + void swap(int iterator, account_name payer); + + // gets a prefix that allows for only primary key iterators + static prefix_bundle get_primary_slice_in_primaries(name code, name scope, name table, uint64_t id); + pv_bundle get_primary_key_value(name code, name scope, name table, uint64_t id); + void set_value(const shared_bytes& primary_key, const payer_payload& pp); + int32_t find_and_store_primary_key(const session_variant_type::iterator& session_iter, int32_t table_ei, + const shared_bytes& type_prefix, int32_t not_found_return, + const char* calling_func, uint64_t& found_key); + struct exact_iterator { + bool valid = false; + session_variant_type::iterator itr; + shared_bytes type_prefix; + }; + exact_iterator get_exact_iterator(name code, name scope, name table, uint64_t primary); + + enum class comp { equals, gte, gt}; + int32_t find_i64(name code, name scope, name table, uint64_t id, comp comparison); + + using uint128_t = eosio::chain::uint128_t; + using key256_t = eosio::chain::key256_t; + session_variant_type current_session; + db_key_value_iter_store primary_iter_store; + db_key_value_any_lookup primary_lookup; + db_key_value_sec_lookup sec_lookup_i64; + db_key_value_sec_lookup sec_lookup_i128; + db_key_value_sec_lookup sec_lookup_i256; + db_key_value_sec_lookup sec_lookup_double; + db_key_value_sec_lookup sec_lookup_long_double; + static constexpr uint64_t noop_secondary = 0x0; + }; // db_context_rocksdb + + db_context_rocksdb::db_context_rocksdb(apply_context& context, name receiver, session_variant_type session) + : db_context( context, receiver ), current_session{ session }, primary_lookup(*this, session), + sec_lookup_i64(*this, session), sec_lookup_i128(*this, session), sec_lookup_i256(*this, session), + sec_lookup_double(*this, session), sec_lookup_long_double(*this, session) {} + + db_context_rocksdb::~db_context_rocksdb() { + } + + int32_t db_context_rocksdb::db_store_i64(uint64_t scope, uint64_t table, account_name payer, uint64_t id, const char* value , size_t value_size) { + EOS_ASSERT( payer != account_name(), invalid_table_payer, "must specify a valid account to pay for new record" ); + const name scope_name{scope}; + const name table_name{table}; + const auto old_key_value = get_primary_key_value(receiver, scope_name, table_name, id); + + EOS_ASSERT( !old_key_value.value, db_rocksdb_invalid_operation_exception, "db_store_i64 called with pre-existing key"); + + primary_lookup.add_table_if_needed(old_key_value.full_key, payer); + + const payer_payload pp{payer, value, value_size}; + set_value(old_key_value.full_key, pp); + + const int64_t billable_size = static_cast(value_size + db_key_value_any_lookup::overhead); + + std::string event_id; + auto dm_logger = context.control.get_deep_mind_logger(); + if (dm_logger != nullptr) { + event_id = db_context::table_event(receiver, scope_name, table_name, name(id)); + } + + update_db_usage( payer, billable_size, db_context::row_add_trace(context.get_action_id(), std::move(event_id)) ); + + if (dm_logger != nullptr) { + db_context::log_row_insert(*dm_logger, context.get_action_id(), receiver, scope_name, table_name, payer, + name(id), value, value_size); + } + + const unique_table t { receiver, scope_name, table_name }; + const auto table_ei = primary_iter_store.cache_table(t); + return primary_iter_store.add(primary_key_iter(table_ei, id, payer)); + } + + void db_context_rocksdb::db_update_i64(int32_t itr, account_name payer, const char* value , size_t value_size) { + const auto& key_store = primary_iter_store.get(itr); + const auto& table_store = primary_iter_store.get_table(key_store); + EOS_ASSERT( table_store.contract == receiver, table_access_violation, "db access violation" ); + const auto old_key_value = get_primary_key_value(receiver, table_store.scope, table_store.table, key_store.primary); + + EOS_ASSERT( old_key_value.value, db_rocksdb_invalid_operation_exception, + "invariant failure in db_update_i64, iter store found to update but nothing in database"); + + // copy locally, since below the key_store memory will be changed + const auto old_payer = key_store.payer; + if (payer.empty()) { + payer = old_payer; + } + + const payer_payload old_pp{*old_key_value.value}; + const auto old_value_actual_size = old_pp.value_size; + + const payer_payload pp{payer, value, value_size}; + set_value(old_key_value.full_key, pp); + + std::string event_id; + auto dm_logger = context.control.get_deep_mind_logger(); + if (dm_logger != nullptr) { + event_id = db_context::table_event(table_store.contract, table_store.scope, table_store.table, name(key_store.primary)); + } + + const int64_t overhead = db_key_value_any_lookup::overhead; + const int64_t old_size = static_cast(old_value_actual_size + overhead); + const int64_t new_size = static_cast(value_size + overhead); + + if( old_payer != payer ) { + // refund the existing payer + update_db_usage( old_payer, -(old_size), db_context::row_update_rem_trace(context.get_action_id(), std::string(event_id)) ); + // charge the new payer + update_db_usage( payer, (new_size), db_context::row_update_add_trace(context.get_action_id(), std::move(event_id)) ); + + // swap the payer in the iterator store + swap(itr, payer); + } else if(old_size != new_size) { + // charge/refund the existing payer the difference + update_db_usage( old_payer, new_size - old_size, db_context::row_update_trace(context.get_action_id(), std::move(event_id)) ); + } + + if (dm_logger != nullptr) { + db_context::log_row_update(*dm_logger, context.get_action_id(), table_store.contract, table_store.scope, + table_store.table, old_payer, payer, name(key_store.primary), + old_key_value.value->data(), old_key_value.value->size(), value, value_size); + } + } + + void db_context_rocksdb::db_remove_i64(int32_t itr) { + const auto& key_store = primary_iter_store.get(itr); + const auto& table_store = primary_iter_store.get_table(key_store); + EOS_ASSERT( table_store.contract == receiver, table_access_violation, "db access violation" ); + const auto old_key_value = get_primary_key_value(receiver, table_store.scope, table_store.table, key_store.primary); + + EOS_ASSERT( old_key_value.value, db_rocksdb_invalid_operation_exception, + "invariant failure in db_remove_i64, iter store found to update but nothing in database"); + + const auto old_payer = key_store.payer; + + std::string event_id; + auto dm_logger = context.control.get_deep_mind_logger(); + if (dm_logger != nullptr) { + event_id = db_context::table_event(table_store.contract, table_store.scope, table_store.table, name(key_store.primary)); + } + + payer_payload pp(*old_key_value.value); + update_db_usage( old_payer, -(pp.value_size + db_key_value_any_lookup::overhead), db_context::row_rem_trace(context.get_action_id(), std::move(event_id)) ); + + if (dm_logger != nullptr) { + db_context::log_row_remove(*dm_logger, context.get_action_id(), table_store.contract, table_store.scope, + table_store.table, old_payer, name(key_store.primary), pp.value, + pp.value_size); + } + + current_session.erase(old_key_value.full_key); + primary_lookup.remove_table_if_empty(old_key_value.full_key); + + primary_iter_store.remove(itr); // don't use key_store anymore + } + + int32_t db_context_rocksdb::db_get_i64(int32_t itr, char* value , size_t value_size) { + const auto& key_store = primary_iter_store.get(itr); + const auto& table_store = primary_iter_store.get_table(key_store); + const auto old_key_value = get_primary_key_value(table_store.contract, table_store.scope, table_store.table, key_store.primary); + + EOS_ASSERT( old_key_value.value, db_rocksdb_invalid_operation_exception, + "invariant failure in db_get_i64, iter store found to update but nothing in database"); + payer_payload pp {*old_key_value.value}; + const char* const actual_value = pp.value; + const size_t actual_size = pp.value_size; + if (value_size == 0) { + return actual_size; + } + const size_t copy_size = std::min(value_size, actual_size); + memcpy( value, actual_value, copy_size ); + return copy_size; + } + + int32_t db_context_rocksdb::db_next_i64(int32_t itr, uint64_t& primary) { + if (itr < primary_iter_store.invalid_iterator()) return primary_iter_store.invalid_iterator(); // cannot increment past end iterator of table + + const auto& key_store = primary_iter_store.get(itr); + const auto& table_store = primary_iter_store.get_table(key_store); + auto exact = + get_exact_iterator(table_store.contract, table_store.scope, table_store.table, key_store.primary); + EOS_ASSERT( exact.valid, db_rocksdb_invalid_operation_exception, + "invariant failure in db_next_i64, iter store found to update but it does not exist in the database"); + auto& session_iter = exact.itr; + auto& type_prefix = exact.type_prefix; + ++session_iter; + return find_and_store_primary_key(session_iter, key_store.table_ei, type_prefix, + key_store.table_ei, __func__, primary); + } + + int32_t db_context_rocksdb::db_previous_i64(int32_t itr, uint64_t& primary) { + if( itr < primary_iter_store.invalid_iterator() ) { // is end iterator + const backing_store::unique_table* table_store = primary_iter_store.find_table_by_end_iterator(itr); + EOS_ASSERT( table_store, invalid_table_iterator, "not a valid end iterator" ); + if (current_session.begin() == current_session.end()) { + // NOTE: matching chainbase functionality, if iterator store found, but no keys in db + return primary_iter_store.invalid_iterator(); + } + + const auto primary_bounded_key = + get_primary_slice_in_primaries(table_store->contract, table_store->scope, table_store->table, + std::numeric_limits::max()); + auto session_iter = current_session.lower_bound(primary_bounded_key.full_key); + + auto past_end = [&](const auto& iter) { + return !primary_lookup.match_prefix(primary_bounded_key.prefix_key, iter); + }; + // if we are at the end of the db or past the end of this table, then step back one + if (past_end(session_iter)) { + --session_iter; + // either way, the iterator after our known table should have a primary key in this table as the previous iterator + if (past_end(session_iter)) { + // NOTE: matching chainbase functionality, if iterator store found, but no key in db for table + return primary_iter_store.invalid_iterator(); + } + } + + return find_and_store_primary_key(session_iter, primary_iter_store.get_end_iterator_by_table(*table_store), + primary_bounded_key.prefix_key, primary_iter_store.invalid_iterator(), + __func__, primary); + } + + const auto& key_store = primary_iter_store.get(itr); + const backing_store::unique_table& table_store = primary_iter_store.get_table(key_store); + const auto slice_primary_key = get_primary_slice_in_primaries(table_store.contract, table_store.scope, table_store.table, key_store.primary); + auto exact = get_exact_iterator(table_store.contract, table_store.scope, table_store.table, key_store.primary); + auto& session_iter = exact.itr; + // if we didn't find the key, or know that we cannot decrement the iterator, then return the invalid iterator handle + if (!exact.valid || session_iter == current_session.begin()) { + return primary_iter_store.invalid_iterator(); + } + + auto& type_prefix = exact.type_prefix; + --session_iter; + return find_and_store_primary_key(session_iter, key_store.table_ei, type_prefix, + primary_iter_store.invalid_iterator(), __func__, primary); + } + + int32_t db_context_rocksdb::db_find_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) { + return find_i64(name{code}, name{scope}, name{table}, id, comp::equals); + } + + int32_t db_context_rocksdb::db_lowerbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) { + return find_i64(name{code}, name{scope}, name{table}, id, comp::gte); + } + + int32_t db_context_rocksdb::db_upperbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) { + return find_i64(name{code}, name{scope}, name{table}, id, comp::gt); + } + + int32_t db_context_rocksdb::db_end_i64(uint64_t code, uint64_t scope, uint64_t table) { + return primary_lookup.get_end_iter(name{code}, name{scope}, name{table}, primary_iter_store); + } + + /** + * interface for uint64_t secondary + */ + int32_t db_context_rocksdb::db_idx64_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, + const uint64_t& secondary) { + return sec_lookup_i64.store(name{scope}, name{table}, payer, id, secondary); + } + + void db_context_rocksdb::db_idx64_update(int32_t iterator, account_name payer, const uint64_t& secondary) { + sec_lookup_i64.update(iterator, payer, secondary); + } + + void db_context_rocksdb::db_idx64_remove(int32_t iterator) { + sec_lookup_i64.remove(iterator); + } + + int32_t db_context_rocksdb::db_idx64_find_secondary(uint64_t code, uint64_t scope, uint64_t table, const uint64_t& secondary, + uint64_t& primary) { + return sec_lookup_i64.find_secondary(name{code}, name{scope}, name{table}, secondary, primary); + } + + int32_t db_context_rocksdb::db_idx64_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, + uint64_t primary) { + return sec_lookup_i64.find_primary(name{code}, name{scope}, name{table}, secondary, primary); + } + + int32_t db_context_rocksdb::db_idx64_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, + uint64_t& primary) { + return sec_lookup_i64.lowerbound_secondary(name{code}, name{scope}, name{table}, secondary, primary); + } + + int32_t db_context_rocksdb::db_idx64_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, + uint64_t& primary) { + return sec_lookup_i64.upperbound_secondary(name{code}, name{scope}, name{table}, secondary, primary); + } + + int32_t db_context_rocksdb::db_idx64_end(uint64_t code, uint64_t scope, uint64_t table) { + return sec_lookup_i64.end_secondary(name{code}, name{scope}, name{table}); + } + + int32_t db_context_rocksdb::db_idx64_next(int32_t iterator, uint64_t& primary) { + return sec_lookup_i64.next_secondary(iterator, primary); + } + + int32_t db_context_rocksdb::db_idx64_previous(int32_t iterator, uint64_t& primary) { + return sec_lookup_i64.previous_secondary(iterator, primary); + } + + /** + * interface for uint128_t secondary + */ + int32_t db_context_rocksdb::db_idx128_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, + const uint128_t& secondary) { + return sec_lookup_i128.store(name{scope}, name{table}, payer, id, secondary); + } + + void db_context_rocksdb::db_idx128_update(int32_t iterator, account_name payer, const uint128_t& secondary) { + sec_lookup_i128.update(iterator, payer, secondary); + } + + void db_context_rocksdb::db_idx128_remove(int32_t iterator) { + sec_lookup_i128.remove(iterator); + } + + int32_t db_context_rocksdb::db_idx128_find_secondary(uint64_t code, uint64_t scope, uint64_t table, const uint128_t& secondary, + uint64_t& primary) { + return sec_lookup_i128.find_secondary(name{code}, name{scope}, name{table}, secondary, primary); + } + + int32_t db_context_rocksdb::db_idx128_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, + uint64_t primary) { + return sec_lookup_i128.find_primary(name{code}, name{scope}, name{table}, secondary, primary); + } + + int32_t db_context_rocksdb::db_idx128_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, + uint64_t& primary) { + return sec_lookup_i128.lowerbound_secondary(name{code}, name{scope}, name{table}, secondary, primary); + } + + int32_t db_context_rocksdb::db_idx128_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, + uint64_t& primary) { + return sec_lookup_i128.upperbound_secondary(name{code}, name{scope}, name{table}, secondary, primary); + } + + int32_t db_context_rocksdb::db_idx128_end(uint64_t code, uint64_t scope, uint64_t table) { + return sec_lookup_i128.end_secondary(name{code}, name{scope}, name{table}); + } + + int32_t db_context_rocksdb::db_idx128_next(int32_t iterator, uint64_t& primary) { + return sec_lookup_i128.next_secondary(iterator, primary); + } + + int32_t db_context_rocksdb::db_idx128_previous(int32_t iterator, uint64_t& primary) { + return sec_lookup_i128.previous_secondary(iterator, primary); + } + + eosio::chain::key256_t convert(const uint128_t* data) { + eosio::chain::key256_t secondary; + std::memcpy(secondary.data(), data, sizeof(secondary)); + return secondary; + } + + void convert_back(uint128_t* data, const eosio::chain::key256_t& secondary) { + std::memcpy(data, secondary.data(), sizeof(secondary)); + } + + /** + * interface for 256-bit interger secondary + */ + int32_t db_context_rocksdb::db_idx256_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, + const uint128_t* data) { + const auto secondary = convert(data); + return sec_lookup_i256.store(name{scope}, name{table}, payer, id, secondary); + } + + void db_context_rocksdb::db_idx256_update(int32_t iterator, account_name payer, const uint128_t* data) { + const auto secondary = convert(data); + sec_lookup_i256.update(iterator, payer, secondary); + } + + void db_context_rocksdb::db_idx256_remove(int32_t iterator) { + sec_lookup_i256.remove(iterator); + } + + int32_t db_context_rocksdb::db_idx256_find_secondary(uint64_t code, uint64_t scope, uint64_t table, const uint128_t* data, + uint64_t& primary) { + const auto secondary = convert(data); + return sec_lookup_i256.find_secondary(name{code}, name{scope}, name{table}, secondary, primary); + } + + int32_t db_context_rocksdb::db_idx256_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, + uint64_t primary) { + auto secondary = convert(data); + auto ret = sec_lookup_i256.find_primary(name{code}, name{scope}, name{table}, secondary, primary); + convert_back(data, secondary); + return ret; + } + + int32_t db_context_rocksdb::db_idx256_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, + uint64_t& primary) { + auto secondary = convert(data); + auto ret = sec_lookup_i256.lowerbound_secondary(name{code}, name{scope}, name{table}, secondary, primary); + convert_back(data, secondary); + return ret; + } + + int32_t db_context_rocksdb::db_idx256_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, + uint64_t& primary) { + auto secondary = convert(data); + auto ret = sec_lookup_i256.upperbound_secondary(name{code}, name{scope}, name{table}, secondary, primary); + convert_back(data, secondary); + return ret; + } + + int32_t db_context_rocksdb::db_idx256_end(uint64_t code, uint64_t scope, uint64_t table) { + return sec_lookup_i256.end_secondary(name{code}, name{scope}, name{table}); + } + + int32_t db_context_rocksdb::db_idx256_next(int32_t iterator, uint64_t& primary) { + return sec_lookup_i256.next_secondary(iterator, primary); + } + + int32_t db_context_rocksdb::db_idx256_previous(int32_t iterator, uint64_t& primary) { + return sec_lookup_i256.previous_secondary(iterator, primary); + } + + /** + * interface for double secondary + */ + int32_t db_context_rocksdb::db_idx_double_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, + const float64_t& secondary) { + return sec_lookup_double.store(name{scope}, name{table}, payer, id, secondary); + } + + void db_context_rocksdb::db_idx_double_update(int32_t iterator, account_name payer, const float64_t& secondary) { + sec_lookup_double.update(iterator, payer, secondary); + } + + void db_context_rocksdb::db_idx_double_remove(int32_t iterator) { + sec_lookup_double.remove(iterator); + } + + int32_t db_context_rocksdb::db_idx_double_find_secondary(uint64_t code, uint64_t scope, uint64_t table, + const float64_t& secondary, uint64_t& primary) { + return sec_lookup_double.find_secondary(name{code}, name{scope}, name{table}, secondary, primary); + } + + int32_t db_context_rocksdb::db_idx_double_find_primary(uint64_t code, uint64_t scope, uint64_t table, float64_t& secondary, + uint64_t primary) { + return sec_lookup_double.find_primary(name{code}, name{scope}, name{table}, secondary, primary); + } + + int32_t db_context_rocksdb::db_idx_double_lowerbound(uint64_t code, uint64_t scope, uint64_t table, float64_t& secondary, + uint64_t& primary) { + return sec_lookup_double.lowerbound_secondary(name{code}, name{scope}, name{table}, secondary, primary); + } + + int32_t db_context_rocksdb::db_idx_double_upperbound(uint64_t code, uint64_t scope, uint64_t table, float64_t& secondary, + uint64_t& primary) { + return sec_lookup_double.upperbound_secondary(name{code}, name{scope}, name{table}, secondary, primary); + } + + int32_t db_context_rocksdb::db_idx_double_end(uint64_t code, uint64_t scope, uint64_t table) { + return sec_lookup_double.end_secondary(name{code}, name{scope}, name{table}); + } + + int32_t db_context_rocksdb::db_idx_double_next(int32_t iterator, uint64_t& primary) { + return sec_lookup_double.next_secondary(iterator, primary); + } + + int32_t db_context_rocksdb::db_idx_double_previous(int32_t iterator, uint64_t& primary) { + return sec_lookup_double.previous_secondary(iterator, primary); + } + + /** + * interface for long double secondary + */ + int32_t db_context_rocksdb::db_idx_long_double_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, + const float128_t& secondary) { + return sec_lookup_long_double.store(name{scope}, name{table}, payer, id, secondary); + } + + void db_context_rocksdb::db_idx_long_double_update(int32_t iterator, account_name payer, const float128_t& secondary) { + sec_lookup_long_double.update(iterator, payer, secondary); + } + + void db_context_rocksdb::db_idx_long_double_remove(int32_t iterator) { + sec_lookup_long_double.remove(iterator); + } + + int32_t db_context_rocksdb::db_idx_long_double_find_secondary(uint64_t code, uint64_t scope, uint64_t table, + const float128_t& secondary, uint64_t& primary) { + return sec_lookup_long_double.find_secondary(name{code}, name{scope}, name{table}, secondary, primary); + } + + int32_t db_context_rocksdb::db_idx_long_double_find_primary(uint64_t code, uint64_t scope, uint64_t table, + float128_t& secondary, uint64_t primary) { + return sec_lookup_long_double.find_primary(name{code}, name{scope}, name{table}, secondary, primary); + } + + int32_t db_context_rocksdb::db_idx_long_double_lowerbound(uint64_t code, uint64_t scope, uint64_t table, float128_t& secondary, + uint64_t& primary) { + return sec_lookup_long_double.lowerbound_secondary(name{code}, name{scope}, name{table}, secondary, primary); + } + + int32_t db_context_rocksdb::db_idx_long_double_upperbound(uint64_t code, uint64_t scope, uint64_t table, float128_t& secondary, + uint64_t& primary) { + return sec_lookup_long_double.upperbound_secondary(name{code}, name{scope}, name{table}, secondary, primary); + } + + int32_t db_context_rocksdb::db_idx_long_double_end(uint64_t code, uint64_t scope, uint64_t table) { + return sec_lookup_long_double.end_secondary(name{code}, name{scope}, name{table}); + } + + int32_t db_context_rocksdb::db_idx_long_double_next(int32_t iterator, uint64_t& primary) { + return sec_lookup_long_double.next_secondary(iterator, primary); + } + + int32_t db_context_rocksdb::db_idx_long_double_previous(int32_t iterator, uint64_t& primary) { + return sec_lookup_long_double.previous_secondary(iterator, primary); + } + + // for primary keys in the iterator store, we don't care about secondary key + db_context_rocksdb::prim_key_iter_type db_context_rocksdb::primary_key_iter(int table_ei, uint64_t key, account_name payer) { + return prim_key_iter_type { .table_ei = table_ei, .secondary = noop_secondary, .primary = key, .payer = payer}; + } + + void db_context_rocksdb::swap(int iterator, account_name payer) { + primary_iter_store.swap(iterator, noop_secondary, payer); + } + + // gets a prefix that allows for only primary key iterators + prefix_bundle db_context_rocksdb::get_primary_slice_in_primaries(name code, name scope, name table, uint64_t id) { + bytes primary_key = db_key_value_format::create_primary_key(scope, table, id); + return { primary_key, end_of_prefix::at_type, code }; + } + + pv_bundle db_context_rocksdb::get_primary_key_value(name code, name scope, name table, uint64_t id) { + prefix_bundle primary_key = get_primary_slice_in_primaries(code, scope, table, id); + return { primary_key, current_session.read(primary_key.full_key) }; + } + + void db_context_rocksdb::set_value(const shared_bytes& primary_key, const payer_payload& pp) { + current_session.write(primary_key, pp.as_payload()); + } + + int32_t db_context_rocksdb::find_and_store_primary_key(const session_variant_type::iterator& session_iter, + int32_t table_ei, const shared_bytes& type_prefix, + int32_t not_found_return, const char* calling_func, + uint64_t& found_key) { + // if nothing remains in the database, return the passed in value + if (session_iter == current_session.end() || !primary_lookup.match_prefix(type_prefix, (*session_iter).first)) { + return not_found_return; + } + EOS_ASSERT( db_key_value_format::get_primary_key((*session_iter).first, type_prefix, found_key), db_rocksdb_invalid_operation_exception, + "invariant failure in ${func}, iter store found to update but no primary keys in database", + ("func", calling_func)); + + const account_name found_payer = payer_payload(*(*session_iter).second).payer; + + return primary_iter_store.add(primary_key_iter(table_ei, found_key, found_payer)); + } + + // returns the exact iterator and the bounding key (type) + db_context_rocksdb::exact_iterator db_context_rocksdb::get_exact_iterator( + name code, name scope, name table, uint64_t primary) { + auto slice_primary_key = get_primary_slice_in_primaries(code, scope, table, primary); + auto session_iter = current_session.lower_bound(slice_primary_key.full_key); + const bool valid = primary_lookup.match(slice_primary_key.full_key, session_iter); + return { .valid = valid, .itr = std::move(session_iter), .type_prefix = std::move(slice_primary_key.prefix_key) }; + } + + int32_t db_context_rocksdb::find_i64(name code, name scope, name table, uint64_t id, comp comparison) { + // expanding the "in-play" iterator space to include every key type for that table, to ensure we know if + // the key is not found, that there is anything in the table at all (and thus can return an end iterator + // or if an invalid iterator needs to be returned + prefix_bundle primary_and_prefix_keys { db_key_value_format::create_primary_key(scope, table, id), + end_of_prefix::pre_type, code }; + auto session_iter = current_session.lower_bound(primary_and_prefix_keys.full_key); + auto is_in_table = [&prefix_key=primary_and_prefix_keys.prefix_key, + &primary_lookup=this->primary_lookup](const session_variant_type::iterator& iter) { + return primary_lookup.match_prefix(prefix_key, iter); + }; + if (!is_in_table(session_iter)) { + // if no iterator was found in this table, then there is no table entry, so the table is empty + return primary_iter_store.invalid_iterator(); + } + + const unique_table t { code, scope, table }; + const auto table_ei = primary_iter_store.cache_table(t); + + const auto& desired_type_prefix = + db_key_value_format::create_full_key_prefix(primary_and_prefix_keys.full_key, end_of_prefix::at_type); + + std::optional primary_key; + if (!primary_lookup.match(primary_and_prefix_keys.full_key, (*session_iter).first)) { + if (comparison == comp::equals) { + return table_ei; + } + else if (!primary_lookup.match_prefix(desired_type_prefix, (*session_iter).first)) { + return table_ei; + } + } + else if (comparison == comp::gt) { + ++session_iter; + // if we don't have a match, we need to identify if we went past the primary types, and thus are at the end + EOS_ASSERT( is_in_table(session_iter), db_rocksdb_invalid_operation_exception, + "invariant failure in find_i64, primary key found but was not followed by a table key"); + if (!primary_lookup.match_prefix(desired_type_prefix, session_iter)) { + return table_ei; + } + } + else { + // since key is exact, and we didn't advance, it is id + primary_key = id; + } + + const account_name found_payer = payer_payload(*(*session_iter).second).payer; + if (!primary_key) { + uint64_t key = 0; + const auto valid = db_key_value_format::get_primary_key((*session_iter).first, desired_type_prefix, key); + EOS_ASSERT( valid, db_rocksdb_invalid_operation_exception, + "invariant failure in find_i64, primary key already verified but method indicates it is not a primary key"); + primary_key = key; + } + + return primary_iter_store.add(primary_key_iter(table_ei, *primary_key, found_payer)); + } + + std::unique_ptr create_db_rocksdb_context(apply_context& context, name receiver, + db_context_rocksdb::session_variant_type session) + { + static_assert(std::is_convertible::value, "cannot convert"); + static_assert(std::is_convertible, std::default_delete >::value, "cannot convert delete"); + return std::make_unique(context, receiver, std::move(session)); + } + +}}} // namespace eosio::chain::backing_store diff --git a/libraries/chain/backing_store/db_key_value_any_lookup.cpp b/libraries/chain/backing_store/db_key_value_any_lookup.cpp new file mode 100644 index 00000000000..f6506376bfc --- /dev/null +++ b/libraries/chain/backing_store/db_key_value_any_lookup.cpp @@ -0,0 +1,128 @@ +#include +#include +#include +#include +#include +#include +#include + +namespace eosio { namespace chain { namespace backing_store { + + eosio::session::shared_bytes make_useless_value() { + const char null = '\0'; + return eosio::session::shared_bytes {&null, 1}; + } + + const eosio::session::shared_bytes db_key_value_any_lookup::useless_value = make_useless_value(); + + void db_key_value_any_lookup::add_table_if_needed(const shared_bytes& key, account_name payer) { + auto table_key = db_key_value_format::create_full_key_prefix(key, end_of_prefix::pre_type); + auto session_iter = current_session.lower_bound(table_key); + if (!match_prefix(table_key, session_iter)) { + // need to add the key_type::table to the end + table_key = db_key_value_format::create_table_key(table_key); + const auto legacy_key = db_key_value_format::extract_legacy_key(table_key); + const auto extracted_data = db_key_value_format::get_prefix_thru_key_type(legacy_key); + std::string event_id; + apply_context& context = parent.context; + const auto& scope = std::get<0>(extracted_data); + const auto& table = std::get<1>(extracted_data); + auto dm_logger = context.control.get_deep_mind_logger(); + if (dm_logger != nullptr) { + event_id = db_context::table_event(parent.receiver, scope, table); + } + + context.update_db_usage(payer, table_overhead, db_context::add_table_trace(context.get_action_id(), std::move(event_id))); + + payer_payload pp(payer, nullptr, 0); + current_session.write(table_key, pp.as_payload()); + + if (dm_logger != nullptr) { + db_context::log_insert_table(*dm_logger, context.get_action_id(), parent.receiver, scope, table, payer); + } + } + } + + void db_key_value_any_lookup::remove_table_if_empty(const shared_bytes& key) { + // look for any other entries in the table + auto entire_table_prefix_key = db_key_value_format::create_full_key_prefix(key, end_of_prefix::pre_type); + // since this prefix key is just scope and table, it will include all primary, secondary, and table keys + auto session_itr = current_session.lower_bound(entire_table_prefix_key); + EOS_ASSERT( session_itr != current_session.end(), db_rocksdb_invalid_operation_exception, + "invariant failure in remove_table_if_empty, iter store found and removed, but no table entry was found"); + auto key_value = *session_itr; + EOS_ASSERT( match_prefix(entire_table_prefix_key, key_value.first), db_rocksdb_invalid_operation_exception, + "invariant failure in remove_table_if_empty, iter store found and removed, but no table entry was found"); + // check if the only entry for this contract/scope/table is the table entry + auto legacy_key = db_key_value_format::extract_legacy_key(key_value.first); + if (db_key_value_format::extract_key_type(legacy_key) != backing_store::db_key_value_format::key_type::table) { + return; + } + + const auto extracted_data = db_key_value_format::get_prefix_thru_key_type(legacy_key); + const auto& scope = std::get<0>(extracted_data); + const auto& table = std::get<1>(extracted_data); + + const name payer = payer_payload{*key_value.second}.payer; + std::string event_id; + apply_context& context = parent.context; + auto dm_logger = context.control.get_deep_mind_logger(); + if (dm_logger != nullptr) { + event_id = db_context::table_event(parent.receiver, scope, table); + } + + context.update_db_usage(payer, - table_overhead, db_context::rem_table_trace(context.get_action_id(), std::move(event_id)) ); + + if (dm_logger != nullptr) { + db_context::log_remove_table(*dm_logger, context.get_action_id(), parent.receiver, scope, table, payer); + } + + current_session.erase(key_value.first); + } + + key_bundle db_key_value_any_lookup::get_slice(name code, name scope, name table) { + bytes prefix_key = db_key_value_format::create_prefix_key(scope, table); + return { prefix_key, code }; + } + + key_bundle db_key_value_any_lookup::get_table_end_slice(name code, name scope, name table) { + bytes table_key = db_key_value_format::create_table_key(scope, table); + return { table_key, code }; + } + + bool db_key_value_any_lookup::match_prefix(const shared_bytes& shorter, const shared_bytes& longer) { + if (shorter.size() > longer.size()) { + return false; + } + return memcmp(shorter.data(), longer.data(), shorter.size()) == 0; + } + + bool db_key_value_any_lookup::match_prefix(const shared_bytes& shorter, const session_variant_type::iterator& iter) { + if (iter == current_session.end()) { + return false; + } + return match_prefix(shorter, (*iter).first); + } + + bool db_key_value_any_lookup::match(const shared_bytes& lhs, const shared_bytes& rhs) { + return lhs.size() == rhs.size() && + memcmp(lhs.data(), rhs.data(), lhs.size()) == 0; + } + + bool db_key_value_any_lookup::match(const shared_bytes& lhs, const session_variant_type::iterator& iter) { + if (iter == current_session.end()) { + return false; + } + return match(lhs, (*iter).first); + } + + key_bundle::key_bundle(const b1::chain_kv::bytes& composite_key, name code) + : full_key(db_key_value_format::create_full_key(composite_key, code)){ + } + + prefix_bundle::prefix_bundle(const b1::chain_kv::bytes& composite_key, end_of_prefix prefix_end, name code) + : full_key(db_key_value_format::create_full_key(composite_key, code)), + prefix_key(db_key_value_format::create_full_key_prefix(full_key, prefix_end)) { + } + +}}} // namespace eosio::chain::backing_store diff --git a/libraries/chain/backing_store/db_key_value_format.cpp b/libraries/chain/backing_store/db_key_value_format.cpp new file mode 100644 index 00000000000..0298f0ded8a --- /dev/null +++ b/libraries/chain/backing_store/db_key_value_format.cpp @@ -0,0 +1,293 @@ +#include +#include +#include +#include + +namespace eosio { namespace chain { namespace backing_store { namespace db_key_value_format { + namespace detail { + constexpr std::size_t key_size(key_type kt) { + switch(kt) { + case key_type::primary: + return sizeof(uint64_t); + case key_type::sec_i64: + return sizeof(uint64_t); + case key_type::sec_i128: + return sizeof(eosio::chain::uint128_t); + case key_type::sec_i256: + return sizeof(key256_t); + case key_type::sec_double: + return sizeof(float64_t); + case key_type::sec_long_double: + return sizeof(float128_t); + case key_type::table: + return 0; // a table entry ends at the type and has no trailing sub-"keys" + default: + FC_THROW_EXCEPTION(bad_composite_key_exception, + "DB intrinsic key-value store composite key is malformed, key_size should not be " + "called with key_type: ${kt}", ("kt", to_string(kt))); + } + } + + b1::chain_kv::bytes prepare_composite_key_prefix(name scope, name table, std::size_t type_size, std::size_t key_size, std::size_t extension_size) { + constexpr static auto scope_size = sizeof(scope); + constexpr static auto table_size = sizeof(table); + b1::chain_kv::bytes key_storage; + key_storage.reserve(scope_size + table_size + type_size + key_size + extension_size); + b1::chain_kv::append_key(key_storage, scope.to_uint64_t()); + b1::chain_kv::append_key(key_storage, table.to_uint64_t()); + return key_storage; + } + + b1::chain_kv::bytes prepare_composite_key(name scope, name table, std::size_t key_size, key_type kt, std::size_t extension_size) { + constexpr static auto type_size = sizeof(key_type); + static_assert( type_size == 1, "" ); + auto key_storage = prepare_composite_key_prefix(scope, table, type_size, key_size, extension_size); + key_storage.push_back(static_cast(kt)); + return key_storage; + } + + std::string to_string(const key_type& kt) { + switch (kt) { + case key_type::primary: return "primary key of type uint64_t"; + case key_type::primary_to_sec: return "primary key to secondary key type"; + case key_type::sec_i64: return "secondary key of type uint64_t"; + case key_type::sec_i128: return "secondary key of type uint128_t"; + case key_type::sec_i256: return "secondary key of type key256_t"; + case key_type::sec_double: return "secondary key of type float64_t"; + case key_type::sec_long_double: return "secondary key of type float128_t"; + case key_type::table: return "table end key"; + default: + const int kt_as_int = static_cast(kt); + return std::string(""; + } + } + + intermittent_decomposed_prefix_values extract_from_composite_key_prefix(b1::chain_kv::bytes::const_iterator begin, b1::chain_kv::bytes::const_iterator key_end) { + auto key_loc = begin; + uint64_t scope_name = 0; + EOS_ASSERT(b1::chain_kv::extract_key(key_loc, key_end, scope_name), bad_composite_key_exception, + "DB intrinsic key-value store composite key is malformed, it does not contain a scope"); + uint64_t table_name = 0; + EOS_ASSERT(b1::chain_kv::extract_key(key_loc, key_end, table_name), bad_composite_key_exception, + "DB intrinsic key-value store composite key is malformed, it does not contain a table"); + return { name{scope_name}, name{table_name}, key_loc }; + } + + intermittent_decomposed_values extract_from_composite_key(b1::chain_kv::bytes::const_iterator begin, b1::chain_kv::bytes::const_iterator key_end) { + auto intermittent = extract_from_composite_key_prefix(begin, key_end); + auto& key_loc = std::get<2>(intermittent); + EOS_ASSERT(key_loc != key_end, bad_composite_key_exception, + "DB intrinsic key-value store composite key is malformed, it does not contain an indication of the " + "type of the db-key (primary uint64_t or secondary uint64_t/uint128_t/etc)"); + const key_type kt{*key_loc++}; + return { std::get<0>(intermittent), std::get<1>(intermittent), key_loc, kt }; + } + + // NOTE: very limited use till redesign + constexpr uint64_t db_type_and_code_size = detail::prefix_size() - detail::prefix_size(); // 1 (db type) + 8 (contract) + static_assert(db_type_and_code_size == sizeof(char) + sizeof(name), "Some assumptions on formatting have been broken"); + } // namespace detail + + b1::chain_kv::bytes create_primary_key(name scope, name table, uint64_t primary_key) { + const std::size_t zero_extension_size = 0; + b1::chain_kv::bytes composite_key = detail::prepare_composite_key(scope, table, sizeof(uint64_t), key_type::primary, zero_extension_size); + b1::chain_kv::append_key(composite_key, primary_key); + return composite_key; + } + + bool get_primary_key(const b1::chain_kv::bytes& composite_key, name& scope, name& table, uint64_t& primary_key) { + b1::chain_kv::bytes::const_iterator composite_loc; + key_type kt = key_type::sec_i64; + std::tie(scope, table, composite_loc, kt) = detail::extract_from_composite_key(composite_key.cbegin(), composite_key.cend()); + if (kt != key_type::primary) { + return false; + } + EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_key.cend(), primary_key), bad_composite_key_exception, + "DB intrinsic key-value store composite key is malformed, it is supposed to have a primary key"); + EOS_ASSERT(composite_loc == composite_key.cend(), bad_composite_key_exception, + "DB intrinsic key-value store composite key is malformed, it has extra data after the primary key"); + return true; + } + + b1::chain_kv::bytes create_prefix_type_key(name scope, name table, key_type kt) { + static constexpr std::size_t no_key_size = 0; + static constexpr std::size_t no_extension_size = 0; + b1::chain_kv::bytes composite_key = detail::prepare_composite_key(scope, table, no_key_size, kt, no_extension_size); + return composite_key; + } + + b1::chain_kv::bytes create_table_key(name scope, name table) { + // table key ends with the type, so just reuse method + return create_prefix_type_key(scope, table, key_type::table); + } + + eosio::session::shared_bytes create_table_key(const eosio::session::shared_bytes& prefix_key) { + const std::size_t full_prefix_size = db_key_value_format::detail::prefix_size(); + EOS_ASSERT(prefix_key.size() >= full_prefix_size, bad_composite_key_exception, + "DB intrinsic key-value store composite key is malformed, create_table_key expects to be given a full " + "key that has a full prefix leading up to the type"); + const auto new_kt = static_cast(key_type::table); + const char* new_kt_ptr = &new_kt; + // already have everything in char format/order, so just need to assemble it + return detail::assemble<2, eosio::session::shared_bytes>({rocksdb::Slice{prefix_key.data(), full_prefix_size}, + rocksdb::Slice{new_kt_ptr, 1}}); + } + + bool get_table_key(const b1::chain_kv::bytes& composite_key, name& scope, name& table) { + b1::chain_kv::bytes::const_iterator composite_loc; + key_type kt = key_type::primary; + std::tie(scope, table, composite_loc, kt) = detail::extract_from_composite_key(composite_key.cbegin(), composite_key.cend()); + if (kt != key_type::table) { + return false; + } + EOS_ASSERT(composite_loc == composite_key.cend(), bad_composite_key_exception, + "DB intrinsic key-value store composite key is malformed, there should be no trailing primary/secondary key"); + return true; + } + + b1::chain_kv::bytes create_prefix_key(name scope, name table) { + static constexpr std::size_t no_type_size = 0; + static constexpr std::size_t no_key_size = 0; + static constexpr std::size_t no_extension_size = 0; + b1::chain_kv::bytes composite_key = detail::prepare_composite_key_prefix(scope, table, no_type_size, no_key_size, no_extension_size); + return composite_key; + } + + void get_prefix_key(const b1::chain_kv::bytes& composite_key, name& scope, name& table) { + b1::chain_kv::bytes::const_iterator composite_loc; + std::tie(scope, table, composite_loc) = detail::extract_from_composite_key_prefix(composite_key.cbegin(), composite_key.cend()); + } + + bool get_trailing_primary_key(const rocksdb::Slice& full_key, const rocksdb::Slice& secondary_key_prefix, uint64_t& primary_key) { + const auto sec_prefix_size = secondary_key_prefix.size(); + EOS_ASSERT(full_key.size() == sec_prefix_size + sizeof(primary_key), bad_composite_key_exception, + "DB intrinsic key-value get_trailing_primary_key was passed a full key size: ${s1} bytes that was not " + "exactly ${s2} bytes (the size of a primary key) larger than the secondary_key_prefix size: ${s3}", + ("s1", full_key.size())("s2", sizeof(primary_key))("s3", sec_prefix_size)); + const auto comp = memcmp(secondary_key_prefix.data(), full_key.data(), sec_prefix_size); + if (comp != 0) { + return false; + } + auto start_offset = full_key.data() + sec_prefix_size; + const b1::chain_kv::bytes composite_primary_key {start_offset, start_offset + sizeof(primary_key)}; + auto composite_loc = composite_primary_key.cbegin(); + EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_primary_key.cend(), primary_key), bad_composite_key_exception, + "DB intrinsic key-value store invariant has changed, extract_key should only fail if the string size is" + " less than the sizeof(uint64_t)"); + return true; + } + + key_type extract_key_type(const b1::chain_kv::bytes& composite_key) { + key_type kt = key_type::primary; + std::tie(std::ignore, std::ignore, std::ignore, kt) = detail::extract_from_composite_key(composite_key.cbegin(), composite_key.cend()); + return kt; + } + + key_type extract_primary_to_sec_key_type(const b1::chain_kv::bytes& composite_key) { + b1::chain_kv::bytes::const_iterator composite_loc; + key_type kt = key_type::primary; + std::tie(std::ignore, std::ignore, composite_loc, kt) = detail::extract_from_composite_key(composite_key.cbegin(), composite_key.cend()); + EOS_ASSERT(kt == key_type::primary_to_sec, bad_composite_key_exception, + "DB intrinsic extract_primary_to_sec_key_type was passed a key that was not of type: ${type}", ("type", detail::to_string(kt))); + EOS_ASSERT(composite_loc != composite_key.cend(), bad_composite_key_exception, + "DB intrinsic extract_primary_to_sec_key_type was passed a key that was only the type prefix"); + const key_type sec_kt{*composite_loc++}; + return sec_kt; + } + + intermittent_decomposed_values get_prefix_thru_key_type(const b1::chain_kv::bytes& composite_key) { + return detail::extract_from_composite_key(composite_key.cbegin(), composite_key.cend()); + } + + b1::chain_kv::bytes extract_legacy_key(const eosio::session::shared_bytes& complete_key) { + const auto slice = extract_legacy_slice(complete_key); + b1::chain_kv::bytes ret {slice.data(), slice.data() + slice.size()}; + return ret; + } + + rocksdb::Slice extract_legacy_slice(const eosio::session::shared_bytes& full_key) { + // offset into the whole key for the old postfix (after db type and contract) + return {full_key.data() + detail::db_type_and_code_size, + full_key.size() - detail::db_type_and_code_size}; + } + + eosio::session::shared_bytes create_full_key(const b1::chain_kv::bytes& composite_key, name code) { + static const char db_type_prefix = make_rocksdb_contract_db_prefix(); + b1::chain_kv::bytes code_as_bytes; + b1::chain_kv::append_key(code_as_bytes, code.to_uint64_t()); + auto ret = eosio::session::make_shared_bytes({std::string_view{&db_type_prefix, 1}, + std::string_view{code_as_bytes.data(), + code_as_bytes.size()}, + std::string_view{composite_key.data(), + composite_key.size()}}); + return ret; + } + + eosio::session::shared_bytes create_full_key_prefix(const eosio::session::shared_bytes& full_key, end_of_prefix prefix_end) { + auto calc_extension = [](end_of_prefix prefix_end) { + switch (prefix_end) { + case end_of_prefix::pre_type: return std::size_t(0); + case end_of_prefix::at_type: return std::size_t(1); + case end_of_prefix::at_prim_to_sec_type: return std::size_t(2); + case end_of_prefix::at_prim_to_sec_primary_key: return 2 + sizeof(uint64_t); + } + }; + const std::size_t extension_size = calc_extension(prefix_end); + const std::size_t full_prefix_size = db_key_value_format::detail::prefix_size() + extension_size; + EOS_ASSERT( full_prefix_size < full_key.size(), db_rocksdb_invalid_operation_exception, + "invariant failure in prefix_bundle, the passed in full_key was: ${size1} bytes, but it needs to " + "be at least: ${size2}", ("size1", full_key.size())("size2", full_prefix_size)); + return eosio::session::shared_bytes(full_key.data(), full_prefix_size); + } + + full_key_data parse_full_key(const eosio::session::shared_bytes& full_key) { + static const char db_type_prefix = make_rocksdb_contract_db_prefix(); + EOS_ASSERT( full_key.size() >= 1, db_rocksdb_invalid_operation_exception, + "parse_full_key was passed an empty key."); + full_key_data data; + const char* offset = full_key.data(); + data.db_type = *(offset++); + const std::size_t db_type_size = 1; + if (data.db_type != db_type_prefix || full_key.size() == db_type_size) { + return data; + } + const std::size_t full_prefix_size = db_key_value_format::detail::prefix_size(); + const std::size_t full_prefix_type_size = full_prefix_size + sizeof(key_type); + const std::size_t db_type_and_contract_size = db_type_size + sizeof(name); + EOS_ASSERT( full_key.size() >= db_type_and_contract_size, db_rocksdb_invalid_operation_exception, + "parse_full_key was passed a key with a db type and erroneous data trailing."); + const b1::chain_kv::bytes composite_contract_key {offset, offset + sizeof(name)}; + offset += sizeof(name); + auto composite_loc = composite_contract_key.cbegin(); + uint64_t contract; + EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_contract_key.cend(), contract), bad_composite_key_exception, + "DB intrinsic key-value store invariant has changed, extract_key should only fail if the string size is" + " less than the sizeof(uint64_t), which was verified that it was not"); + EOS_ASSERT( composite_loc == composite_contract_key.cend(), db_rocksdb_invalid_operation_exception, + "bytes constructed to extract contract was constructed incorrectly."); + data.contract = name{contract}; + const auto remaining = full_key.size() - db_type_and_contract_size; + if (!remaining) { + return data; + } + data.legacy_key = b1::chain_kv::bytes {offset, offset + remaining}; + auto prefix_thru_kt = get_prefix_thru_key_type(*data.legacy_key); + data.scope = std::get<0>(prefix_thru_kt); + data.table = std::get<1>(prefix_thru_kt); + data.kt = std::get<3>(prefix_thru_kt); + const std::size_t type_prefix_length = std::distance(data.legacy_key->cbegin(), std::get<2>(prefix_thru_kt)); + data.type_prefix = {data.legacy_key->data(), type_prefix_length}; + + return data; + } + + eosio::session::shared_bytes create_full_primary_key(name code, name scope, name table, uint64_t primary_key) { + bytes composite_key = create_primary_key(scope, table, primary_key); + return create_full_key(composite_key, code); + } + + eosio::session::shared_bytes create_full_prefix_key(name code, name scope, name table, std::optional kt) { + bytes composite_key = kt ? create_prefix_type_key(scope, table, *kt) : create_prefix_key(scope, table); + return create_full_key(composite_key, code); + } +}}}} // namespace eosio::chain::backing_store::db_key_value_format diff --git a/libraries/chain/backing_store/kv_context.cpp b/libraries/chain/backing_store/kv_context.cpp new file mode 100644 index 00000000000..88f8f9fa747 --- /dev/null +++ b/libraries/chain/backing_store/kv_context.cpp @@ -0,0 +1,54 @@ +#include +#include +#include +#include + +namespace eosio { namespace chain { + + namespace { + void kv_resource_manager_update_ram(apply_context& context, int64_t delta, const kv_resource_trace& trace, account_name payer) { + std::string event_id; + if (context.control.get_deep_mind_logger() != nullptr) { + event_id = STORAGE_EVENT_ID("${id}", ("id", fc::to_hex(trace.key.data(), trace.key.size()))); + } + + context.update_db_usage(payer, delta, storage_usage_trace(context.get_action_id(), std::move(event_id), "kv", trace.op_to_string())); + } + } + kv_resource_manager create_kv_resource_manager(apply_context& context) { + return {&context, config::billable_size_v, &kv_resource_manager_update_ram}; + } + + /* + int64_t kv_context::create_table_usage(kv_resource_manager& resource_manager, const account_name& payer, const char* key, const uint32_t key_size, const uint32_t value_size) { + const int64_t resource_delta = (static_cast(resource_manager.billable_size) + key_size + value_size); + resource_manager.update_table_usage(payer, resource_delta, kv_resource_trace(key, key_size, kv_resource_trace::operation::create)); + return resource_delta; + } + + int64_t kv_context::erase_table_usage(kv_resource_manager& resource_manager, const account_name& payer, const char* key, const uint32_t key_size, const uint32_t value_size) { + const int64_t resource_delta = -(static_cast(resource_manager.billable_size) + key_size + value_size); + resource_manager.update_table_usage(payer, resource_delta, kv_resource_trace(key, key_size, kv_resource_trace::operation::erase)); + return resource_delta; + } + + int64_t kv_context::update_table_usage(kv_resource_manager& resource_manager, const account_name& old_payer, const account_name& new_payer, const char* key, const uint32_t key_size, const uint32_t old_value_size, const uint32_t new_value_size) { + // 64-bit arithmetic cannot overflow, because both the key and value are limited to 32-bits + const int64_t old_size = key_size + old_value_size; + const int64_t new_size = key_size + new_value_size; + const int64_t resource_delta = new_size - old_size; + + if (old_payer != new_payer) { + // refund the existing payer + resource_manager.update_table_usage(old_payer, -(old_size + resource_manager.billable_size), kv_resource_trace(key, key_size, kv_resource_trace::operation::update)); + // charge the new payer for full amount + resource_manager.update_table_usage(new_payer, new_size + resource_manager.billable_size, kv_resource_trace(key, key_size, kv_resource_trace::operation::update)); + } else if (old_size != new_size) { + // adjust delta for the existing payer + resource_manager.update_table_usage(new_payer, resource_delta, kv_resource_trace(key, key_size, kv_resource_trace::operation::update)); + } // No need for a final "else" as usage does not change + + return resource_delta; + } + */ +}} diff --git a/libraries/chain/backing_store/tests/CMakeLists.txt b/libraries/chain/backing_store/tests/CMakeLists.txt new file mode 100644 index 00000000000..ea982e3e443 --- /dev/null +++ b/libraries/chain/backing_store/tests/CMakeLists.txt @@ -0,0 +1,11 @@ +add_executable( test_kv_rocksdb test_kv_rocksdb.cpp ) +target_link_libraries( test_kv_rocksdb eosio_chain fc chainbase ) +target_include_directories( test_kv_rocksdb PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/../include") +add_test(NAME test_kv_rocksdb COMMAND libraries/chain/backing_store/tests/test_kv_rocksdb WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + +add_executable( benchmark_kv benchmark_kv.cpp ) +target_link_libraries( benchmark_kv eosio_chain fc chainbase Boost::program_options ) +target_include_directories( benchmark_kv PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/../include") +file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/benchmark_kv_batch.py DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) +file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/benchmark_kv_single.py DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) +file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/README DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) diff --git a/libraries/chain/backing_store/tests/README b/libraries/chain/backing_store/tests/README new file mode 100644 index 00000000000..fc80e2696b7 --- /dev/null +++ b/libraries/chain/backing_store/tests/README @@ -0,0 +1,23 @@ +Microbenchmarking instructions + +1. Run "./benchmark_kv -h" to get all options. +2. To prepare a random key file, + a). use "pwgen -1" to generate + "number-of-keys" keys with "key-length", "-1" indicating + one key per line in a file. + b). use "pwgen" multile times to generate different key sized + files and concatenate them. + c). use "sort -u" to remove duplicate keys. + d). use "shuf" to randomize the final file. +3. To prepare a workset file, use "shuf -n". +4. To facilitate running a suite of benchmarking, "benchmark_kv.py" + is provided. You need to prepare "data" directory which + contains a set of key files and their workset files. + a). run "benchmark_kv.py" will benchmark operations + (get, egt_data, set, create, erase, it_create, it_next, + it_key_value) with all key files on rocksdb and chainkv. + b). a key file must end with ".keys", its workset files must + start with the key file's main part and end with ".ws". + For example, "100k_random.keys" is a key file, + "100k_random_100.ws" and "100k_random_500.ws" are its + workset files. diff --git a/libraries/chain/backing_store/tests/benchmark_kv.cpp b/libraries/chain/backing_store/tests/benchmark_kv.cpp new file mode 100644 index 00000000000..5a7639f297b --- /dev/null +++ b/libraries/chain/backing_store/tests/benchmark_kv.cpp @@ -0,0 +1,599 @@ +#include +#include + +#include +#include +#include + +#include +#include + +#include +#include + +using namespace eosio; +using namespace eosio::chain; + +namespace kv_benchmark { + +// Global test data +constexpr account_name receiver = "kvrdb"_n; +constexpr uint64_t contract = receiver.to_uint64_t(); +constexpr account_name payer = "payer"_n; +constexpr uint64_t default_billable_size = 12; + +struct cmd_args { + std::string operation = ""; + std::string backing_store = ""; + std::string key_file = ""; + std::string workset_file = ""; + uint32_t value_size = 1024; + uint64_t num_runs = 1000000; + uint32_t state_size_multiples = 1; // For Chainbase. Multiples of 1GB +}; + +struct measurement_t { + double user_duration_us_avg; + double system_duration_us_avg; + uint64_t actual_num_runs; + uint32_t minor_faults; + uint32_t major_faults; + uint32_t blocks_in; + uint32_t blocks_out; +}; + +struct dummy_control { + fc::logger* get_deep_mind_logger() { + return nullptr; + } + + uint32_t get_action_id() { + return 0; + } +}; + +struct dummy_context { + dummy_control control; + + uint32_t get_action_id() { + return 0; + } +}; + +struct mock_resource_manager { + uint64_t billable_size = default_billable_size; + + int64_t update_table_usage(account_name payer, int64_t delta, const kv_resource_trace& trace) { + return 0; + } + + dummy_context* _context; +}; + +// resource_manager, receiver, limits in kv_context are +// initialzed as a reference. Place them globally to avoid +// going out of scope +constexpr uint32_t max_key_size = 100; +constexpr uint32_t max_value_size = 1020*1024; +constexpr uint32_t max_iterators = 100; +kv_database_config limits { max_key_size, max_value_size, max_iterators }; +mock_resource_manager resource_manager; + +// Main purpose of this program is to microbenchmark individual +// KV API over RocksDB and ChainKV. + +// Loads keys and returns number of keys as an argument. +// This is mainly used to benchmark key creation and erase. +std::vector load_keys(const cmd_args& args, uint32_t& num_keys) { + std::vector keys; + std::ifstream key_file(args.key_file); + + if (!key_file.is_open()) { + std::cerr << "Failed to open key file " << args.key_file << std::endl; + exit(2); + } + + std::string key; + num_keys = 0; + while (getline(key_file, key)) { + keys.push_back(key); + ++num_keys; + } + + key_file.close(); + + return keys; +} + +// Load workset. Benchmarking "get", "get_data", and "set" +// requires it. +std::vector load_workset(const cmd_args& args, uint32_t& workset_size) { + std::ifstream workset_ifs(args.workset_file); + + if (!workset_ifs.is_open()) { + std::cerr << "Failed to open workset file " << args.workset_file << std::endl; + exit(2); + } + + std::string key; + std::vector workset; + workset_size = 0; + while (getline(workset_ifs, key)) { + ++workset_size; + workset.push_back(key); + } + + workset_ifs.close(); + return workset; +} + +// Read keys from key file and create key-values pairs +// on backing store. +void create_key_values(const cmd_args& args, const std::unique_ptr& kv_context_ptr, uint32_t& num_keys) { + std::ifstream key_file(args.key_file); + + if (!key_file.is_open()) { + std::cerr << "Failed to open key file " << args.key_file << std::endl; + exit(2); + } + + std::string value(args.value_size, 'a'); + std::string key; + num_keys = 0; + while (getline(key_file, key)) { + kv_context_ptr->kv_set(contract, key.c_str(), key.size(), value.c_str(), value.size(), payer); + ++num_keys; + } + + key_file.close(); +} + +// Returns the number of loops required to perform total_runs +// for runs_per_loop +uint32_t get_num_loops(const uint64_t total_runs, const uint32_t runs_per_loop) { + if (total_runs <= runs_per_loop) { + return 1; // one loop is sufficient + } else { + return total_runs/runs_per_loop + 1 ; + } +} + +// Returns difference between two timevals +double time_diff_us(const timeval& start, const timeval& end) { + timeval diff; + timersub(&end, &start, &diff); + return (1000000 * diff.tv_sec + diff.tv_usec); +} + +// Returns calculated measurement based on raw data +measurement_t calculated_measurement(const uint64_t actual_num_runs, const rusage& usage_start, const rusage& usage_end) { + measurement_t m; + + m.actual_num_runs = actual_num_runs; + m.user_duration_us_avg = time_diff_us(usage_start.ru_utime, usage_end.ru_utime)/actual_num_runs; + m.system_duration_us_avg = time_diff_us(usage_start.ru_stime, usage_end.ru_stime)/actual_num_runs; + m.minor_faults = uint64_t(usage_end.ru_minflt - usage_start.ru_minflt); + m.major_faults = uint64_t(usage_end.ru_majflt - usage_start.ru_majflt); + m.blocks_in = uint64_t(usage_end.ru_inblock - usage_start.ru_inblock); + m.blocks_out = uint64_t(usage_end.ru_oublock - usage_start.ru_oublock); + + return m; +} + +// Benchmark "get" operation +measurement_t benchmark_get(const cmd_args& args, const std::unique_ptr& kv_context_ptr, const std::vector& workset) { + uint32_t num_loops = get_num_loops(args.num_runs, workset.size()); + rusage usage_start, usage_end; + + getrusage(RUSAGE_SELF, &usage_start); + for (auto i = 0U; i < num_loops; ++i) { + for (auto& key: workset) { + uint32_t actual_value_size; + kv_context_ptr->kv_get(contract, key.c_str(), key.size(),actual_value_size); + } + } + getrusage(RUSAGE_SELF, &usage_end); + + return calculated_measurement(num_loops*workset.size(), usage_start, usage_end); +} + +// Benchmark "get_data" operation +measurement_t benchmark_get_data(const cmd_args& args, const std::unique_ptr& kv_context_ptr, const std::vector& workset) { + char* data = new char[args.value_size]; + uint32_t num_loops = get_num_loops(args.num_runs, workset.size()); + rusage usage_start, usage_end; + + getrusage(RUSAGE_SELF, &usage_start); + for (auto i = 0U; i < num_loops; ++i) { + for (auto& key: workset) { + kv_context_ptr->kv_get_data(0, data, args.value_size); + } + } + getrusage(RUSAGE_SELF, &usage_end); + + return calculated_measurement(num_loops*workset.size(), usage_start, usage_end); +} + +// Benchmark "set" operation +measurement_t benchmark_set(const cmd_args& args, const std::unique_ptr& kv_context_ptr, const std::vector& workset) { + std::string value(args.value_size, 'b'); + uint32_t num_loops = get_num_loops(args.num_runs, workset.size()); + rusage usage_start, usage_end; + + getrusage(RUSAGE_SELF, &usage_start); + for (auto i = 0U; i < num_loops; ++i) { + for (auto& key: workset) { + kv_context_ptr->kv_set(contract, key.c_str(), key.size(), value.c_str(), value.size(), payer); + } + } + getrusage(RUSAGE_SELF, &usage_end); + + return calculated_measurement(num_loops*workset.size(), usage_start, usage_end); +} + +// Benchmark "create" operation +measurement_t benchmark_create(const cmd_args& args, const std::unique_ptr& kv_context_ptr, uint32_t& num_keys) { + // read keys from the key file + std::vector keys = load_keys(args, num_keys); + std::string value(args.value_size, 'a'); + rusage usage_start, usage_end; + + getrusage(RUSAGE_SELF, &usage_start); + for (auto& key: keys) { + kv_context_ptr->kv_set(contract, key.c_str(), key.size(), value.c_str(), value.size(), payer); + } + getrusage(RUSAGE_SELF, &usage_end); + + return calculated_measurement(keys.size(), usage_start, usage_end); +} + +// Benchmark "erase" operation +measurement_t benchmark_erase(const cmd_args& args, const std::unique_ptr& kv_context_ptr, uint32_t& num_keys) { + // create keys in backing store first + std::vector keys = load_keys(args, num_keys); + std::string value(args.value_size, 'a'); + for (auto& key: keys) { + kv_context_ptr->kv_set(contract, key.c_str(), key.size(), value.c_str(), value.size(), payer); + } + + rusage usage_start, usage_end; + getrusage(RUSAGE_SELF, &usage_start); + for (auto& key: keys) { + kv_context_ptr->kv_erase(contract, key.c_str(), key.size()); + } + getrusage(RUSAGE_SELF, &usage_end); + + return calculated_measurement(keys.size(), usage_start, usage_end); +} + +// Benchmark "it_create" operation +measurement_t benchmark_it_create(const cmd_args& args, const std::unique_ptr& kv_context_ptr) { + rusage usage_start, usage_end; + getrusage(RUSAGE_SELF, &usage_start); + auto i = 0U; + std::string prefix = "a"; + while (i < args.num_runs) { + kv_context_ptr->kv_it_create(contract, prefix.c_str(), 0); // kv_it_create creates a unique pointer. Will be destoryed at the end of the scope. + ++i; + } + getrusage(RUSAGE_SELF, &usage_end); + + return calculated_measurement(args.num_runs, usage_start, usage_end); +} + +// Benchmark "it_next" operation +measurement_t benchmark_it_next(const cmd_args& args, const std::unique_ptr& kv_context_ptr, uint32_t num_keys) { + uint32_t found_key_size, found_value_size; + auto it = kv_context_ptr->kv_it_create(contract, "", 0); + it->kv_it_next(&found_key_size, &found_value_size); + + rusage usage_start, usage_end; + getrusage(RUSAGE_SELF, &usage_start); + while (it->kv_it_status() != kv_it_stat::iterator_end) { + it->kv_it_next(&found_key_size, &found_value_size); + } + getrusage(RUSAGE_SELF, &usage_end); + + // As we are iterate the whole set of the keys, the number of runs + // is num_keys. + return calculated_measurement(num_keys, usage_start, usage_end); +} + +// Benchmark "it_key" operation +measurement_t benchmark_it_key(const cmd_args& args, const std::unique_ptr& kv_context_ptr, uint32_t num_keys) { + uint32_t offset = 0; + char* dest = new char[args.value_size]; + uint32_t actual_size; + uint32_t found_key_size, found_value_size; + + auto it = kv_context_ptr->kv_it_create(contract, "", 0); + it->kv_it_next(&found_key_size, &found_value_size); // move to the first position + + rusage usage_start, usage_end; + getrusage(RUSAGE_SELF, &usage_start); + while (it->kv_it_status() != kv_it_stat::iterator_end) { + it->kv_it_key(offset, dest, found_key_size, actual_size); + it->kv_it_next(&found_key_size, &found_value_size); + } + getrusage(RUSAGE_SELF, &usage_end); + + // As we are iterate the whole set of the keys, the number of runs + // is num_keys. + return calculated_measurement(num_keys, usage_start, usage_end); +} + +// Benchmark "it_value" operation +measurement_t benchmark_it_value(const cmd_args& args, const std::unique_ptr& kv_context_ptr, uint32_t num_keys) { + uint32_t offset = 0; + char* dest = new char[args.value_size]; + uint32_t actual_size; + uint32_t found_key_size, found_value_size; + + auto it = kv_context_ptr->kv_it_create(contract, "", 0); + it->kv_it_next(&found_key_size, &found_value_size); // move to the first position + + rusage usage_start, usage_end; + getrusage(RUSAGE_SELF, &usage_start); + while (it->kv_it_status() != kv_it_stat::iterator_end) { + it->kv_it_value(offset, dest, found_value_size, actual_size); + it->kv_it_next(&found_key_size, &found_value_size); + } + getrusage(RUSAGE_SELF, &usage_end); + + // As we are iterate the whole set of the keys, the number of runs + // is num_keys. + return calculated_measurement(num_keys, usage_start, usage_end); +} + +// Print out benchmarking results +void print_results(const cmd_args& args, const uint32_t num_keys, const uint32_t workset_size, const measurement_t& m) { + std::cout + << "backing_store: " << args.backing_store + << ", operation: " << args.operation + << ", key_file: " << args.key_file + << ", num_keys: " << num_keys + << ", workset_file: " << args.workset_file + << ", workset_size: " << workset_size + << ", value_size: " << args.value_size + << ", num_runs: " << m.actual_num_runs + << ", user_cpu_us_avg: " << m.user_duration_us_avg + << ", system_cpu_us_avg: " << m.system_duration_us_avg + << ", minor_faults_total: " << m.minor_faults + << ", major_faults_total: " << m.major_faults + << ", blocks_in_total: " << m.blocks_in + << ", blocks_out_total: " << m.blocks_out + << std::endl; +} + +// Dispatcher to benchmark individual operation +void benchmark_operation(const cmd_args& args, const std::unique_ptr& kv_context_ptr) { + measurement_t m; + uint32_t num_keys {0}, workset_size {0}; + + // Benchmarking "create" and "erase" creates key values + // on their own + if (args.operation != "create" && args.operation != "erase") { + create_key_values(args, kv_context_ptr, num_keys); + } + + if (args.operation == "get" || args.operation == "get_data" || args.operation == "set") { + std::vector workset = load_workset(args, workset_size); + + if (args.operation == "get" ) { + m = benchmark_get(args, std::move(kv_context_ptr), workset); + } else if (args.operation == "get_data" ) { + m = benchmark_get_data(args, std::move(kv_context_ptr), workset); + } else if (args.operation == "set" ) { + m = benchmark_set(args, std::move(kv_context_ptr), workset); + } + } else { + // workset is not used. + if (args.operation == "it_create" ) { + m = benchmark_it_create(args, std::move(kv_context_ptr)); + } else if (args.operation == "it_next" ) { + m = benchmark_it_next(args, std::move(kv_context_ptr), num_keys); + } else if (args.operation == "it_key" ) { + m = benchmark_it_key(args, std::move(kv_context_ptr), num_keys); + } else if (args.operation == "it_value" ) { + m = benchmark_it_value(args, std::move(kv_context_ptr), num_keys); + } else if (args.operation == "create" ) { + m = benchmark_create(args, std::move(kv_context_ptr), num_keys); + } else if (args.operation == "erase" ) { + m = benchmark_erase(args, std::move(kv_context_ptr), num_keys); + } else { + std::cerr << "Unknown operation: " << args.operation << std::endl; + exit(3); + } + }; + + print_results(args, num_keys, workset_size, m); +} + +inline std::shared_ptr make_rocks_db(const std::string& name) { + rocksdb::DB* cache_ptr{ nullptr }; + auto cache = std::shared_ptr{}; + + auto options = rocksdb::Options{}; + options.create_if_missing = true; // Creates a database if it is missing + options.level_compaction_dynamic_level_bytes = true; + options.bytes_per_sync = 1048576; // used to control the write rate of flushes and compactions. + + // By default, RocksDB uses only one background thread + // for flush and compaction. + // Good value for `total_threads` is the number of cores + options.IncreaseParallelism(7); + + options.OptimizeLevelStyleCompaction(512ull << 20); // optimizes level style compaction + + // Number of open files that can be used by the DB. + // Setting it to -1 means files opened are always kept open. + options.max_open_files = -1; + + // Use this option to increase the number of threads + // used to open the files. + options.max_file_opening_threads = 7; // Default should be the # of Cores + + // Write Buffer Size - Sets the size of a single + // memtable. Once memtable exceeds this size, it is + // marked immutable and a new one is created. + // Default should be 128MB + options.write_buffer_size = 128 * 1024 * 1024; + options.max_write_buffer_number = 10; // maximum number of memtables, both active and immutable + options.min_write_buffer_number_to_merge = 2; // minimum number of memtables to be merged before flushing to storage + + // Once level 0 reaches this number of files, L0->L1 compaction is triggered. + options.level0_file_num_compaction_trigger = 2; + + // Size of L0 = write_buffer_size * min_write_buffer_number_to_merge * level0_file_num_compaction_trigger + // therefore: L0 in stable state = 128MB * 2 * 2 = 512MB + // max_bytes_for_level_basei is total size of level 1. + // For optimal performance make this equal to L0 hence 512MB + options.max_bytes_for_level_base = 512 * 1024 * 1024; + + // Files in level 1 will have target_file_size_base + // bytes. It’s recommended setting target_file_size_base + // to be max_bytes_for_level_base / 10, + // so that there are 10 files in level 1.i.e. 512MB/10 + options.target_file_size_base = (512 * 1024 * 1024) / 10; + + // This value represents the maximum number of threads + // that will concurrently perform a compaction job by + // breaking it into multiple, + // smaller ones that are run simultaneously. + options.max_subcompactions = 7; // Default should be the # of CPUs + + // Full and partitioned filters in the block-based table + // use an improved Bloom filter implementation, enabled + // with format_version 5 (or above) because previous + // releases cannot read this filter. This replacement is + // faster and more accurate, especially for high bits + // per key or millions of keys in a single (full) filter. + rocksdb::BlockBasedTableOptions table_options; + table_options.format_version = 5; + table_options.index_block_restart_interval = 16; + + // Sets the bloom filter - Given an arbitrary key, + // this bit array may be used to determine if the key + // may exist or definitely does not exist in the key set. + table_options.filter_policy.reset(rocksdb::NewBloomFilterPolicy(15, false)); + table_options.index_type = rocksdb::BlockBasedTableOptions::kBinarySearch; + + // Incorporates the Table options into options + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + auto status = rocksdb::DB::Open(options, name.c_str(), &cache_ptr); + + cache.reset(cache_ptr); + + return cache; +} + +// The driver +void benchmark(const cmd_args& args) { + if (args.backing_store == "rocksdb") { + boost::filesystem::remove_all("kvrdb-tmp"); // Use a clean RocksDB + boost::filesystem::remove_all(chain::config::default_state_dir_name); + + constexpr size_t max_rocks_iterators = 1024; + auto rocks_session = eosio::session::make_session(make_rocks_db("kvrdb-tmp"), max_rocks_iterators); + auto session = eosio::session::session{rocks_session}; + + std::unique_ptr kv_context_ptr = create_kv_rocksdb_context(session, receiver, resource_manager, limits); + benchmark_operation(args, std::move(kv_context_ptr)); // kv_context_ptr must be in the same scope as kv_db and usage_start, since they are references in create_kv_rocksdb_context + } else { + boost::filesystem::remove_all(chain::config::default_state_dir_name); // Use a clean Chainbase + chainbase::database chainbase_db(chain::config::default_state_dir_name, database::read_write, args.state_size_multiples * chain::config::default_state_size); // Default is 1024*1024*1024ll == 1073741824 + chainbase_db.add_index(); + + std::unique_ptr kv_context_ptr = create_kv_chainbase_context(chainbase_db, receiver, resource_manager, limits); + benchmark_operation(args, std::move(kv_context_ptr)); + } +} +} // namespace kv_benchmark + +namespace bpo = boost::program_options; +using bpo::options_description; +using bpo::variables_map; + +int main(int argc, char* argv[]) { + kv_benchmark::cmd_args args; + + variables_map vmap; + options_description cli ("kv_benchmark command line options"); + + std::string gts; + cli.add_options() + ("key-file,k", bpo::value()->required(), "the file storing all the keys, mandatory") + ("workset,w", bpo::value(), "the file storing workset keys, which must be constructed from key-file and be random; the operation is repeatedly run against the workset; mandatory for get, get_data, and set") + ("operation,o", bpo::value()->required(), "operation to be benchmarked: get, get_data, set, create, erase, it_create, it_next, it_key, or it_value, mandatory") + ("backing-store,b", bpo::value()->required(), "the database where kay vlaues are stored, rocksdb or chainbase, mandatory") + ("value-size,v", bpo::value(), "value size for the keys") + ("state-size-multiples,s", bpo::value(), "multiples of 1GB for Chainbase state storage") + ("num-runs,n", bpo::value(), "minimum number of runs of the benchmarked operation") + ("help,h","microbenchmarks KV operations get, get_data, set, create (set to a new key), erase, it_create, it_next, it_key, and it_value against chainbase and rocksdb. Please note: numbers in it_key and it_value include those in it_next"); + + try { + bpo::store(bpo::parse_command_line(argc, argv, cli), vmap); + bpo::notify(vmap); + + if (vmap.count("help") > 0) { + cli.print(std::cerr); + return 0; + } + + if (vmap.count("key-file") > 0) { + args.key_file = vmap["key-file"].as(); + } + if (vmap.count("workset") > 0) { + args.workset_file = vmap["workset"].as(); + } + if (vmap.count("operation") > 0) { + args.operation = vmap["operation"].as(); + if (args.operation != "get" && args.operation != "get_data" && args.operation != "set" && args.operation != "create" && args.operation != "erase" && args.operation != "it_create" && args.operation != "it_next" && args.operation != "it_key" && args.operation != "it_value") { + std::cerr << "\'--operation\' must be get, get_data, set, create, erase, it_create, it_next, it_key, or it_value" << std::endl; + return 1; + } + } + if (vmap.count("value-size") > 0) { + args.value_size = vmap["value-size"].as(); + } + if (vmap.count("state-size-multiples") > 0) { + args.state_size_multiples = vmap["state-size-multiples"].as(); + } + if (vmap.count("num-runs") > 0) { + args.num_runs = vmap["num-runs"].as(); + } + if (vmap.count("backing-store") > 0) { + args.backing_store = vmap["backing-store"].as(); + + if (args.backing_store != "rocksdb" && args.backing_store != "chainbase") { + std::cerr << "\'--backing-store\' must be rocksdb or chainbase" << std::endl; + return 1; + } + } + } catch (boost::program_options::required_option& ex) { + // This exception is thrown whenever required options are not supplied. + // Need to catch it or we will get a crash. + if (vmap.count("help") == 0) { + // Missing required options is not an exception in "--help" case, + // do not report it as an exception + std::cerr << ex.what() << std::endl; + } + cli.print (std::cerr); + return 1; + } catch (bpo::unknown_option &ex) { + std::cerr << ex.what() << std::endl; + cli.print (std::cerr); + return 1; + } + + if ((args.operation == "get" || args.operation == "get_data" || args.operation == "set") && args.workset_file.empty()) { + std::cerr << "\'--workset\' is required for get, get_data, and set" << std::endl; + cli.print(std::cerr); + return 1; + } + + kv_benchmark::benchmark(args); + + return 0; +} diff --git a/libraries/chain/backing_store/tests/benchmark_kv_batch.py b/libraries/chain/backing_store/tests/benchmark_kv_batch.py new file mode 100755 index 00000000000..abf0b196498 --- /dev/null +++ b/libraries/chain/backing_store/tests/benchmark_kv_batch.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +import subprocess +import glob + +from benchmark_kv_single import runBenchMarking + +""" +This script benchmarks all operations with all key files +in "data" directory on rocksdb and chainbase. +You can add new key files and their workset files to +"data" directory. a key file must end with ".keys", +its workset files must start with the key file's main part +and end with ".ws" +A sample data directory would look like this +10m_random_1k.ws 5m_random_1k.ws 5m_random.keys 5m_sorted_5k.ws +10m_random.keys 5m_random_5k.ws 5m_sorted_1k.ws 5m_sorted.keys +""" +def main(): + key_files = glob.glob('data/*.keys') + for key_file in key_files: + dot_pos = key_file.find('.') + name = key_file[:dot_pos] + workset_files = glob.glob(name+"*.ws") + + for workset in workset_files: + runBenchMarking(key_file, workset); + +if __name__ == "__main__": + main() diff --git a/libraries/chain/backing_store/tests/benchmark_kv_single.py b/libraries/chain/backing_store/tests/benchmark_kv_single.py new file mode 100755 index 00000000000..97bd5900cd0 --- /dev/null +++ b/libraries/chain/backing_store/tests/benchmark_kv_single.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +import sys +import subprocess + +""" +This script benchmarks with a single key file and one workset. +It reports both summary and detailed performance numbers +for easy comparison. +""" + +def runBenchMarking(key_file, workset): + output = [] + print("============================================") + print("Benchmarking starts") + print("============================================\n") + + for op in ["create", "get", "get_data", "set", "erase", "it_create", "it_next", "it_key", "it_value"]: + results = [] + for db in ["rocksdb", "chainbase"]: + cmd = "./benchmark_kv -k " + key_file + " -o " + op + " -b " + db + " -v 512 -s 7" + if op=="get" or op=="get_data" or op=="set": + cmd += " -w " + workset + + proc = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + outs,errs= proc.communicate() + + decoded_outs=outs.decode("utf-8") + print(decoded_outs) + if errs: + print(errs.decode("utf-8")) + results.append(decoded_outs) + + rocksdb = dict(item.split(": ") for item in results[0].split(", ")) + chainbase = dict(item.split(": ") for item in results[1].split(", ")) + out = '{0:<14s}({1:<27} ({2:<}'.format(op, rocksdb["user_cpu_us_avg"] + ", " + rocksdb["system_cpu_us_avg"] + ")", chainbase["user_cpu_us_avg"] + ", " + chainbase["system_cpu_us_avg"] + ")") + output.append(out) + + print("\nSummary:\n") + print("Operation, RocksDB, Chainbase") + print(" (user_cpu_us, sys_cpu_us), (user_cpu_us, sys_cpu_us)\n") + for o in output: + print(o) + +def main(): + key_file = sys.argv[1] + workset = sys.argv[2] + runBenchMarking(key_file, workset); + +if __name__ == "__main__": + main() diff --git a/libraries/chain/backing_store/tests/test_kv_rocksdb.cpp b/libraries/chain/backing_store/tests/test_kv_rocksdb.cpp new file mode 100644 index 00000000000..0f959a9fde6 --- /dev/null +++ b/libraries/chain/backing_store/tests/test_kv_rocksdb.cpp @@ -0,0 +1,682 @@ +#define BOOST_TEST_MODULE kv_rocksdb_unittests + +#include +#include +#include +#include + +#include + +using namespace eosio; +using namespace eosio::chain; + +// Global test data +constexpr account_name receiver = "kvrdb"_n; +constexpr uint64_t contract = receiver.to_uint64_t(); +constexpr account_name payer = "payernam"_n; +std::string prefix = "prefix"; +constexpr uint32_t it_key_size = 10; +constexpr uint64_t default_billable_size = 12; + +// limits in kv_context is initialzed as a reference. +// Declare it globally to avoid going out of scope +constexpr uint32_t max_key_size = 100; +constexpr uint32_t max_value_size = 1020 * 1024; +constexpr uint32_t max_iterators = 3; +kv_database_config limits{ max_key_size, max_value_size, max_iterators }; + +using namespace eosio::chain::backing_store; + +struct kv_rocksdb_fixture { + + class mock_session { + public: + class iterator final { + public: + iterator(kv_rocksdb_fixture& fixture, bool is_end) : m_fixture{ &fixture }, m_is_end{ is_end } {} + iterator(const iterator& it) = default; + iterator(iterator&&) = default; + + iterator& operator=(const iterator& it) = default; + iterator& operator=(iterator&&) = default; + + iterator& operator++() { return *this; } + iterator operator++(int) { return *this; } + iterator& operator--() { m_is_end = false; return *this; } + iterator operator--(int) { return *this; } + std::pair> operator*() const { + return m_fixture->mock_get_kv(); + } + std::pair> operator->() const { + return m_fixture->mock_get_kv(); + } + bool operator==(const iterator& other) const { return m_is_end == other.m_is_end; } + bool operator!=(const iterator& other) const { return !(*this == other); } + + bool deleted() const { return m_fixture->mock_is_erased(); } + eosio::session::shared_bytes key() { return m_fixture->mock_get_kv().first; } + private: + kv_rocksdb_fixture* m_fixture; + bool m_is_end{ false }; + }; + + public: + mock_session(kv_rocksdb_fixture& fixture) : m_fixture{ &fixture } {} + + std::optional read(const eosio::session::shared_bytes& key) const { + return m_fixture->mock_get(key); + } + void write(const eosio::session::shared_bytes& key, const eosio::session::shared_bytes& value) {} + bool contains(const eosio::session::shared_bytes& key) const { return false; } + void erase(const eosio::session::shared_bytes& key) {} + + iterator begin() { return iterator{ *m_fixture, false }; } + iterator end() { return iterator{ *m_fixture, true }; } + iterator lower_bound(const eosio::session::shared_bytes& key) { + return iterator{ *m_fixture, key != m_fixture->mock_get_key_prefix() || m_fixture->mock_is_end() }; + } + + private: + kv_rocksdb_fixture* m_fixture; + }; + + // Dummy + struct mock_resource_manager { + uint64_t billable_size = default_billable_size; + + struct mock_context { + struct mock_control { + fc::logger* get_deep_mind_logger() { return nullptr; } + }; + uint32_t get_action_id() { return 0; } + mock_control control; + }; + mock_context real; + mock_context* _context = ℜ + + int64_t update_table_usage(account_name payer, int64_t delta, const kv_resource_trace& trace) { return 0; } + }; + + kv_rocksdb_fixture() + : session {*this} { + account_name receiver{ "kvrdb"_n }; + mock_resource_manager resource_manager; + + my_kv_context = std::make_unique>(session, receiver, resource_manager, limits); + } + + mock_session session; + + // mock methods + std::function + mock_kv_set; + std::function(const eosio::session::shared_bytes&)> mock_get; + std::function> const()> mock_get_kv; + std::function mock_is_end = [](){return false;}; + std::function mock_is_valid; + std::function mock_is_erased; + std::function mock_get_key_prefix; + + // object being tested + std::unique_ptr> my_kv_context; + + // test case helpers + void check_get_payer(account_name p) { + char buf[backing_store::payer_in_value_size]; + memcpy(buf, &p, backing_store::payer_in_value_size); // copy payer to buffer + BOOST_CHECK(backing_store::get_payer(buf) == p); // read payer from the buffer and should be equal to the original + } + + struct kv_pair { + std::string key; + std::string value; + }; + + enum class it_dir { NEXT, PREV }; + + enum class it_pos { NOT_END, END }; + + enum class it_state { NOT_ERASED, ERASED }; + + enum class it_keys_equal { YES, NO }; + + enum class it_key_existing { YES, NO }; + + enum class it_exception_expected { YES, NO }; + + void check_set_new_value(const kv_pair& kv) { + uint32_t key_size = kv.key.size(); + uint32_t value_size = kv.value.size(); + int64_t resource_delta = default_billable_size + key_size + value_size; + + BOOST_CHECK(my_kv_context->kv_set(contract, kv.key.c_str(), key_size, kv.value.c_str(), value_size, payer) == + resource_delta); + } + + void check_set_existing_value(const kv_pair& kv, uint32_t old_raw_value_size) { + uint32_t key_size = kv.key.size(); + uint32_t value_size = kv.value.size(); + int64_t resource_delta = value_size - (old_raw_value_size - backing_store::payer_in_value_size); + + BOOST_CHECK(my_kv_context->kv_set(contract, kv.key.c_str(), key_size, kv.value.c_str(), value_size, payer) == + resource_delta); + } + + void check_test_kv_it_status(it_state state, it_pos pos, kv_it_stat expected_status) { + mock_get_kv = []() -> std::pair> { + static const auto expected_key = prefix + std::string{"key"}; + static const auto key = eosio::chain::make_prefix_key(contract, expected_key.c_str(), expected_key.size()); + return std::pair{ key, eosio::session::shared_bytes{"payernamValue", 13} }; + }; + mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), prefix.size()); }; + mock_is_erased = []() -> bool { return false; }; + mock_is_end = []() -> bool { return false; }; + std::unique_ptr it = my_kv_context->kv_it_create(contract, prefix.c_str(), prefix.size()); + if (pos != it_pos::END) { + uint32_t found_key_size = 0; + uint32_t found_value_size = 0; + it->kv_it_next(&found_key_size, &found_value_size); + } + mock_is_erased = [state]() -> bool { return state == it_state::ERASED; }; + mock_is_end = [pos]() -> bool { return pos == it_pos::END; }; + BOOST_CHECK(it->kv_it_status() == expected_status); + } + + void check_kv_it_move(it_dir dir, it_state state, it_pos pos, kv_it_stat expected_status) { + mock_is_erased = [state]() -> bool { return state == it_state::ERASED; }; + + mock_is_end = [pos]() -> bool { return pos == it_pos::END; }; + + mock_is_valid = [pos, state]() -> bool { return state == it_state::NOT_ERASED && pos == it_pos::NOT_END; }; + + mock_get_kv = []() -> std::pair> { + return std::pair{ eosio::session::shared_bytes("keykeykeykeykey", 15), + std::optional{eosio::session::shared_bytes("payernamMyvalue", 15)} }; + }; + mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), it_key_size); }; + + std::unique_ptr it = my_kv_context->kv_it_create(contract, prefix.c_str(), it_key_size); + uint32_t found_key_size, found_value_size; + if (state == it_state::ERASED) { + if (dir == it_dir::NEXT) { + BOOST_CHECK_THROW(it->kv_it_next(&found_key_size, &found_value_size), kv_bad_iter); + } else { + BOOST_CHECK_THROW(it->kv_it_prev(&found_key_size, &found_value_size), kv_bad_iter); + } + return; + } + + if (dir == it_dir::NEXT) { + BOOST_CHECK(it->kv_it_next(&found_key_size, &found_value_size) == expected_status); + } else { + BOOST_CHECK(it->kv_it_prev(&found_key_size, &found_value_size) == expected_status); + } + + if (state == it_state::NOT_ERASED && pos == it_pos::NOT_END) { // means valid + BOOST_CHECK(found_key_size == 6); // "key" + BOOST_CHECK(found_value_size == 7); // "Myvalue" + } else { + BOOST_CHECK(found_key_size == 0); + BOOST_CHECK(found_value_size == 0); + } + } + + void check_kv_it_key(it_state state, it_key_existing key_existing, kv_it_stat expected_status) { + mock_is_erased = [state]() -> bool { return state == it_state::ERASED; }; + + std::string key = "keykeykeykeykey"; + constexpr uint32_t key_size = 15; + auto sample_key_value = std::pair{ eosio::session::shared_bytes(key.c_str(), key.size()), + std::optional{eosio::session::shared_bytes("valuevaluevalue", 15)} }; + mock_get_kv = [key_existing, sample_key_value]() -> std::pair> { + if (key_existing == it_key_existing::YES) { + return sample_key_value; + } else { + return {eosio::session::shared_bytes{}, std::optional{}}; + } + }; + mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), it_key_size); }; + + std::unique_ptr it = my_kv_context->kv_it_create(contract, prefix.c_str(), it_key_size); + uint32_t offset = 0; + char dest[key_size]; + uint32_t size = key_size; + uint32_t actual_size; + + if (state == it_state::ERASED) { + BOOST_CHECK_THROW(it->kv_it_key(offset, dest, key_size, actual_size), kv_bad_iter); + return; + } + + if (expected_status != kv_it_stat::iterator_end) { + uint32_t ignore; + it->kv_it_prev(&ignore, &ignore); + } + BOOST_CHECK(it->kv_it_key(offset, dest, size, actual_size) == expected_status); + + if (expected_status == kv_it_stat::iterator_ok) { + BOOST_CHECK(actual_size == 6); + BOOST_CHECK(memcmp(key.c_str(), dest, 6) == 0); + } else { + BOOST_CHECK(actual_size == 0); + } + } + + void check_kv_it_value(it_state state, it_key_existing key_existing, kv_it_stat expected_status) { + mock_is_erased = [state]() -> bool { return state == it_state::ERASED; }; + + std::string value = "payernamMyvalue"; + constexpr uint32_t value_size = 7; // Myvalue + auto sample_key_value = std::pair{ eosio::session::shared_bytes("keykeykeykeykey", 15), + std::optional{eosio::session::shared_bytes(value.c_str(), value.size())} }; + mock_get_kv = [&]() -> std::pair> { + if (key_existing == it_key_existing::YES) { + return sample_key_value; + } else { + return { eosio::session::shared_bytes{}, std::optional{} }; + } + }; + mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), it_key_size); }; + + std::unique_ptr it = my_kv_context->kv_it_create(contract, prefix.c_str(), it_key_size); + uint32_t offset = 0; + char dest[value_size]; + uint32_t actual_size; + + if (state == it_state::ERASED) { + BOOST_CHECK_THROW(it->kv_it_value(offset, dest, value_size, actual_size), kv_bad_iter); + return; + } + + if (expected_status != kv_it_stat::iterator_end) { + uint32_t ignore; + it->kv_it_prev(&ignore, &ignore); + } + BOOST_CHECK(it->kv_it_value(offset, dest, value_size, actual_size) == expected_status); + + if (expected_status == kv_it_stat::iterator_ok) { + BOOST_CHECK(actual_size == value_size); + BOOST_CHECK(memcmp(value.c_str() + backing_store::payer_in_value_size, dest, value_size) == 0); + } else { + BOOST_CHECK(actual_size == 0); + } + } + + void check_kv_it_compare(it_state state, it_exception_expected exception_expected, uint64_t rhs_contract) { + mock_is_erased = [state]() -> bool { return state == it_state::ERASED; }; + mock_get_kv = []() { + return std::pair{ eosio::session::shared_bytes{}, std::optional{} }; + }; + + mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), it_key_size); }; + std::unique_ptr it = my_kv_context->kv_it_create(contract, prefix.c_str(), it_key_size); + + uint32_t num_iterators = 5; + uint32_t size = 10; + mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(rhs_contract, prefix.c_str(), size); }; + kv_iterator_rocksdb rhs{ num_iterators, *my_kv_context->session, rhs_contract, prefix.c_str(), size }; + + if (exception_expected == it_exception_expected::YES) { + BOOST_CHECK_THROW(it->kv_it_compare(rhs), kv_bad_iter); + } else { + BOOST_CHECK(it->kv_it_compare(rhs)==0); + } + } + + void check_kv_it_key_compare(it_state state, it_keys_equal keys_equal, const std::string& mykey) { + mock_is_erased = [state]() -> bool { return state == it_state::ERASED; }; + + mock_get_kv = []() -> std::pair> { + static const auto expected_key = prefix + std::string{"key"}; + static const auto key = eosio::chain::make_prefix_key(contract, expected_key.c_str(), expected_key.size()); + return std::pair{ key, eosio::session::shared_bytes{"payernamValue", 13} }; + }; + mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), prefix.size()); }; + std::unique_ptr it = my_kv_context->kv_it_create(contract, prefix.c_str(), prefix.size()); + if (state == it_state::ERASED) { + BOOST_CHECK_THROW(it->kv_it_key_compare(mykey.c_str(), mykey.size()), kv_bad_iter); + } else { + uint32_t found_key_size = 0; + uint32_t found_value_size = 0; + it->kv_it_next(&found_key_size, &found_value_size); + if (keys_equal == it_keys_equal::YES) { + BOOST_CHECK(it->kv_it_key_compare(mykey.c_str(), mykey.size()) == 0); + } else { + BOOST_CHECK(it->kv_it_key_compare(mykey.c_str(), mykey.size()) != 0); + } + } + } +}; + +// Test cases started +// +BOOST_AUTO_TEST_SUITE(kv_rocksdb_unittests) + +BOOST_AUTO_TEST_CASE(test_actual_value_size) { + BOOST_CHECK(actual_value_size(payer_in_value_size) == 0); + BOOST_CHECK(actual_value_size(payer_in_value_size + 1) == 1); + BOOST_CHECK(actual_value_size(payer_in_value_size + 10) == 10); +} + +BOOST_AUTO_TEST_CASE(test_actual_value_size_small_value) { + // any raw size less than kv_payer_size should throw + BOOST_CHECK_THROW(actual_value_size(payer_in_value_size - 1), kv_rocksdb_bad_value_size_exception); +} + +BOOST_FIXTURE_TEST_CASE(test_get_payer_1_char, kv_rocksdb_fixture) { check_get_payer("a"_n); } + +BOOST_FIXTURE_TEST_CASE(test_get_payer_4_chars, kv_rocksdb_fixture) { check_get_payer("abcd"_n); } + +BOOST_FIXTURE_TEST_CASE(test_get_payer_8_chars, kv_rocksdb_fixture) { check_get_payer("abcdefg"_n); } + +BOOST_AUTO_TEST_CASE(test_actual_value_start) { + char buf[10]; // any size of buffer will work + BOOST_CHECK(actual_value_start(buf) == (buf + payer_in_value_size)); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_erase, kv_rocksdb_fixture) { + std::string v = "111111119876543210"; // "1111111" is payer + mock_get = [v](const eosio::session::shared_bytes&) -> std::optional { + return eosio::session::shared_bytes(v.data(), v.size()); + }; + + std::string key = "key"; + int64_t resource_delta = -(default_billable_size + key.size() + actual_value_size(v.size())); + + BOOST_CHECK(my_kv_context->kv_erase(contract, key.c_str(), key.size()) == resource_delta); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_erase_zero_value, kv_rocksdb_fixture) { + std::string v = "11111111"; // "1111111" is payer, no value existing + mock_get = [v](const eosio::session::shared_bytes&) -> std::optional { + return eosio::session::shared_bytes(v.data(), v.size()); + }; + + std::string key = "key"; + int64_t resource_delta = -(default_billable_size + key.size() + actual_value_size(v.size())); + + BOOST_CHECK(my_kv_context->kv_erase(contract, key.c_str(), key.size()) == resource_delta); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_erase_key_not_exist, kv_rocksdb_fixture) { + mock_get = [](const eosio::session::shared_bytes&) -> std::optional { + return std::optional{}; + }; + + std::string key = "key"; + BOOST_CHECK(my_kv_context->kv_erase(contract, key.c_str(), key.size()) == 0); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_erase_key_contract_not_match, kv_rocksdb_fixture) { + mock_get = [](const eosio::session::shared_bytes&) -> std::optional { + return std::optional{}; + }; + + std::string key = "key"; + BOOST_CHECK_THROW(my_kv_context->kv_erase(contract + 1, key.c_str(), key.size()), table_operation_not_permitted); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_set_new_val, kv_rocksdb_fixture) { + mock_get = [](const eosio::session::shared_bytes&) -> std::optional { + return std::optional{}; // Not found means we need to create new key + }; + + check_set_new_value({ .key = "1", .value = "" }); + check_set_new_value({ .key = "1", .value = "2" }); + check_set_new_value({ .key = "1", .value = "1111222233334444" }); + check_set_new_value({ .key = "1234567890", .value = "1111222233334444" }); + check_set_new_value({ .key = "1234567890", .value = "1111222233334444" }); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_set_bigger_existing_val, kv_rocksdb_fixture) { + std::string v = "111111112345"; + mock_get = [v](const eosio::session::shared_bytes&) -> std::optional { + return eosio::session::shared_bytes(v.data(), v.size()); + }; + + check_set_existing_value({ .key = "1234567890", .value = "23" }, v.size()); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_set_smaller_existing_val, kv_rocksdb_fixture) { + std::string v = "1111111123"; + mock_get = [v](const eosio::session::shared_bytes&) -> std::optional { + return eosio::session::shared_bytes(v.data(), v.size()); + }; + + check_set_existing_value({ .key = "1234567890", .value = "2345678" }, v.size()); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_set_same_existing_val, kv_rocksdb_fixture) { + std::string v = "1111111123"; // First 8 bytes are the payer + mock_get = [v](const eosio::session::shared_bytes&) -> std::optional { + return eosio::session::shared_bytes(v.data(), v.size()); + }; + + check_set_existing_value({ .key = "1234567890", .value = "23" }, v.size()); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_set_key_too_large, kv_rocksdb_fixture) { + std::string key = "key"; + std::string value = "value"; + uint32_t key_size = max_key_size + 1; + uint32_t value_size = value.size(); + + BOOST_CHECK_THROW(my_kv_context->kv_set(contract, key.c_str(), key_size, value.c_str(), value_size, payer), + kv_limit_exceeded); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_set_value_too_large, kv_rocksdb_fixture) { + std::string key = "key"; + std::string value = "value"; + uint32_t key_size = key.size(); + uint32_t value_size = max_value_size + 1; // 1 larger than max + + BOOST_CHECK_THROW(my_kv_context->kv_set(contract, key.c_str(), key_size, value.c_str(), value_size, payer), + kv_limit_exceeded); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_get, kv_rocksdb_fixture) { + std::string v = "1111111123"; // First 8 bytes are the payer + mock_get = [v](const eosio::session::shared_bytes&) -> std::optional { + return eosio::session::shared_bytes(v.data(), v.size()); + }; + + uint32_t value_size; + BOOST_CHECK(my_kv_context->kv_get(contract, "key", 3, value_size)); + BOOST_CHECK(value_size == 2); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_get_not_existing, kv_rocksdb_fixture) { + mock_get = [](const eosio::session::shared_bytes&) -> std::optional { + return std::optional{}; + }; + + uint32_t value_size; + BOOST_CHECK(my_kv_context->kv_get(contract, "key", 3, value_size) == false); + BOOST_CHECK(value_size == 0); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_get_data, kv_rocksdb_fixture) { + std::string v = "111111119876543210"; // First 8 bytes (11111111) are the payer + mock_get = [v](const eosio::session::shared_bytes&) -> std::optional { + return eosio::session::shared_bytes(v.data(), v.size()); + }; + + uint32_t value_size; + constexpr uint32_t data_size = 10; + char data[data_size]; + + // Get the key-value first + BOOST_CHECK(my_kv_context->kv_get(contract, "key", 3, value_size) == true); + + // offset starts at 0 + BOOST_CHECK(my_kv_context->kv_get_data(0, data, data_size) == data_size); + BOOST_CHECK(memcmp(data, v.c_str() + payer_in_value_size, data_size) == 0); + + // offset less than actual data size + BOOST_CHECK(my_kv_context->kv_get_data(5, data, data_size) == data_size); + BOOST_CHECK(memcmp(data, v.c_str() + payer_in_value_size + 5, data_size - 5) == 0); + + // offset greater than actual data size. Data should not be modified + std::string pattern = "mypattern1"; + memcpy(data, pattern.c_str(), data_size); + BOOST_CHECK(my_kv_context->kv_get_data(data_size + 1, data, data_size) == data_size); + BOOST_CHECK(memcmp(data, pattern.c_str(), data_size) == 0); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_get_data_no_key, kv_rocksdb_fixture) { + mock_get = [](const eosio::session::shared_bytes&) -> std::optional { + return std::optional{}; + }; + + uint32_t value_size; + constexpr uint32_t data_size = 10; + char data[data_size]; + std::string pattern = "mypattern1"; + memcpy(data, pattern.c_str(), data_size); + + // Get the key-value first + BOOST_CHECK(my_kv_context->kv_get(contract, "key", 3, value_size) == false); + + BOOST_CHECK(my_kv_context->kv_get_data(0, data, data_size) == 0); + BOOST_CHECK(memcmp(data, pattern.c_str(), data_size) == 0); + BOOST_CHECK(my_kv_context->kv_get_data(100, data, data_size) == 0); // offset too big + BOOST_CHECK(memcmp(data, pattern.c_str(), data_size) == 0); +} + +// Iterators + +BOOST_FIXTURE_TEST_CASE(test_iter_creation_success, kv_rocksdb_fixture) { + mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), it_key_size); }; + BOOST_CHECK_NO_THROW(my_kv_context->kv_it_create(contract, prefix.c_str(), it_key_size)); +} + +BOOST_FIXTURE_TEST_CASE(test_iter_creation_too_many, kv_rocksdb_fixture) { + std::unique_ptr its[max_iterators + 1]; // save first max_iterators so they won't be destroyed + mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), it_key_size); }; + + // Creating max_iterators iterators + for (auto i = 0U; i < max_iterators; ++i) { + its[i] = my_kv_context->kv_it_create(contract, prefix.c_str(), it_key_size); + } + + // Creating one more + BOOST_CHECK_THROW(my_kv_context->kv_it_create(contract, prefix.c_str(), it_key_size), kv_bad_iter); +} + +BOOST_FIXTURE_TEST_CASE(test_iter_creation_key_size_too_big, kv_rocksdb_fixture) { + mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), max_key_size); }; + BOOST_CHECK_NO_THROW(my_kv_context->kv_it_create(contract, prefix.c_str(), max_key_size)); // within the size + BOOST_CHECK_THROW(my_kv_context->kv_it_create(contract, prefix.c_str(), max_key_size + 1), + kv_bad_iter); // outside of the max key size +} + +BOOST_FIXTURE_TEST_CASE(test_is_kv_chainbase_context_iterator, kv_rocksdb_fixture) { + mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), it_key_size); }; + std::unique_ptr it = my_kv_context->kv_it_create(contract, prefix.c_str(), it_key_size); + BOOST_CHECK(!it->is_kv_chainbase_context_iterator()); +} + +BOOST_FIXTURE_TEST_CASE(test_is_kv_rocksdb_context_iterator, kv_rocksdb_fixture) { + mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), it_key_size); }; + std::unique_ptr it = my_kv_context->kv_it_create(contract, prefix.c_str(), it_key_size); + BOOST_CHECK(it->is_kv_rocksdb_context_iterator()); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_it_compare, kv_rocksdb_fixture) { + check_kv_it_compare(it_state::NOT_ERASED, it_exception_expected::NO, contract); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_it_compare_different_contract, kv_rocksdb_fixture) { + check_kv_it_compare(it_state::NOT_ERASED, it_exception_expected::YES, contract + 1); // +1 to make contract different +} + +BOOST_FIXTURE_TEST_CASE(test_kv_it_compare_erased, kv_rocksdb_fixture) { + check_kv_it_compare(it_state::ERASED, it_exception_expected::YES, contract); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_it_key_compare_equal, kv_rocksdb_fixture) { + static const auto key = prefix + std::string{"key"}; + check_kv_it_key_compare(it_state::NOT_ERASED, it_keys_equal::YES, key.c_str()); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_it_key_compare_non_equal, kv_rocksdb_fixture) { + static const auto key = prefix + std::string{"randomkey"}; + check_kv_it_key_compare(it_state::NOT_ERASED, it_keys_equal::NO, key.c_str()); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_it_key_compare_erased, kv_rocksdb_fixture) { + static const auto key = prefix + std::string{"key"}; + check_kv_it_key_compare(it_state::ERASED, it_keys_equal::YES, key.c_str()); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_it_status_end, kv_rocksdb_fixture) { + check_test_kv_it_status(it_state::ERASED, it_pos::END, kv_it_stat::iterator_end); + check_test_kv_it_status(it_state::NOT_ERASED, it_pos::END, kv_it_stat::iterator_end); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_it_status_erased, kv_rocksdb_fixture) { + check_test_kv_it_status(it_state::ERASED, it_pos::NOT_END, kv_it_stat::iterator_erased); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_it_status_ok, kv_rocksdb_fixture) { + check_test_kv_it_status(it_state::NOT_ERASED, it_pos::NOT_END, kv_it_stat::iterator_ok); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_it_move_to_end, kv_rocksdb_fixture) { + mock_is_erased = []() -> bool { return false; }; + mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), it_key_size); }; + std::unique_ptr it = my_kv_context->kv_it_create(contract, prefix.c_str(), it_key_size); + BOOST_CHECK(it->kv_it_move_to_end() == kv_it_stat::iterator_end); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_it_next_erased, kv_rocksdb_fixture) { + check_kv_it_move(it_dir::NEXT, it_state::ERASED, it_pos::NOT_END, kv_it_stat::iterator_ok); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_it_next_noterased_notend, kv_rocksdb_fixture) { + check_kv_it_move(it_dir::NEXT, it_state::NOT_ERASED, it_pos::NOT_END, kv_it_stat::iterator_ok); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_it_next_noterased_end, kv_rocksdb_fixture) { + check_kv_it_move(it_dir::NEXT, it_state::NOT_ERASED, it_pos::END, kv_it_stat::iterator_end); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_it_prev_erased, kv_rocksdb_fixture) { + check_kv_it_move(it_dir::PREV, it_state::ERASED, it_pos::NOT_END, kv_it_stat::iterator_ok); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_it_prev_noterased_notend, kv_rocksdb_fixture) { + check_kv_it_move(it_dir::PREV, it_state::NOT_ERASED, it_pos::NOT_END, kv_it_stat::iterator_ok); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_it_prev_noterased_end, kv_rocksdb_fixture) { + check_kv_it_move(it_dir::PREV, it_state::NOT_ERASED, it_pos::END, kv_it_stat::iterator_end); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_it_key_has_key, kv_rocksdb_fixture) { + check_kv_it_key(it_state::NOT_ERASED, it_key_existing::YES, kv_it_stat::iterator_ok); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_it_key_no_key, kv_rocksdb_fixture) { + check_kv_it_key(it_state::NOT_ERASED, it_key_existing::NO, kv_it_stat::iterator_end); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_it_key_erased, kv_rocksdb_fixture) { + check_kv_it_key(it_state::ERASED, it_key_existing::NO, kv_it_stat::iterator_end); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_it_value_has_key, kv_rocksdb_fixture) { + check_kv_it_value(it_state::NOT_ERASED, it_key_existing::YES, kv_it_stat::iterator_ok); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_it_value_no_key, kv_rocksdb_fixture) { + check_kv_it_value(it_state::NOT_ERASED, it_key_existing::NO, kv_it_stat::iterator_end); +} + +BOOST_FIXTURE_TEST_CASE(test_kv_it_value_erased, kv_rocksdb_fixture) { + check_kv_it_value(it_state::ERASED, it_key_existing::NO, kv_it_stat::iterator_end); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/libraries/chain/block.cpp b/libraries/chain/block.cpp index 91c93f6c359..64e6fc2fdcf 100644 --- a/libraries/chain/block.cpp +++ b/libraries/chain/block.cpp @@ -1,4 +1,5 @@ #include +#include namespace eosio { namespace chain { void additional_block_signatures_extension::reflector_init() { @@ -20,7 +21,30 @@ namespace eosio { namespace chain { } } - flat_multimap signed_block::validate_and_extract_extensions()const { + struct transaction_receipt_translator { + bool legacy = true; + std::variant operator()(const transaction_id_type& tid) const { + return tid; + } + std::variant operator()(const packed_transaction_v0& ptrx) const { + return packed_transaction(ptrx, legacy); + } + std::variant operator()(packed_transaction_v0&& ptrx) const { + return packed_transaction(std::move(ptrx), legacy); + } + }; + + transaction_receipt::transaction_receipt(const transaction_receipt_v0& other, bool legacy) + : transaction_receipt_header(static_cast(other)), + trx(std::visit(transaction_receipt_translator{legacy}, other.trx)) + {} + + transaction_receipt::transaction_receipt(transaction_receipt_v0&& other, bool legacy) + : transaction_receipt_header(std::move(static_cast(other))), + trx(std::visit(transaction_receipt_translator{legacy}, std::move(other.trx))) + {} + + static flat_multimap validate_and_extract_block_extensions(const extensions_type& block_extensions) { using decompose_t = block_extension_types::decompose_t; flat_multimap results; @@ -61,4 +85,71 @@ namespace eosio { namespace chain { } -} } /// namespace eosio::chain \ No newline at end of file + flat_multimap signed_block_v0::validate_and_extract_extensions()const { + return validate_and_extract_block_extensions( block_extensions ); + } + + signed_block::signed_block( const signed_block_v0& other, bool legacy ) + : signed_block_header(static_cast(other)), + prune_state(legacy ? prune_state_type::complete_legacy : prune_state_type::complete), + block_extensions(other.block_extensions) + { + for(const auto& trx : other.transactions) { + transactions.emplace_back(trx, legacy); + } + } + + signed_block::signed_block( signed_block_v0&& other, bool legacy ) + : signed_block_header(std::move(static_cast(other))), + prune_state(legacy ? prune_state_type::complete_legacy : prune_state_type::complete), + block_extensions(std::move(other.block_extensions)) + { + for(auto& trx : other.transactions) { + transactions.emplace_back(std::move(trx), legacy); + } + } + + static std::size_t pruned_trx_receipt_packed_size(const packed_transaction& obj, packed_transaction::cf_compression_type segment_compression) { + return obj.maximum_pruned_pack_size(segment_compression); + } + static std::size_t pruned_trx_receipt_packed_size(const transaction_id_type& obj, packed_transaction::cf_compression_type) { + return fc::raw::pack_size(obj); + } + + std::size_t transaction_receipt::maximum_pruned_pack_size( packed_transaction::cf_compression_type segment_compression ) const { + return fc::raw::pack_size(*static_cast(this)) + 1 + + std::visit([&](const auto& obj){ return pruned_trx_receipt_packed_size(obj, segment_compression); }, trx); + } + + std::size_t signed_block::maximum_pruned_pack_size( packed_transaction::cf_compression_type segment_compression ) const { + std::size_t result = fc::raw::pack_size(fc::unsigned_int(transactions.size())); + for(const transaction_receipt& r: transactions) { + result += r.maximum_pruned_pack_size( segment_compression ); + } + return fc::raw::pack_size(*static_cast(this)) + fc::raw::pack_size(prune_state) + result + fc::raw::pack_size(block_extensions); + } + + flat_multimap signed_block::validate_and_extract_extensions()const { + return validate_and_extract_block_extensions( block_extensions ); + } + + std::optional signed_block::to_signed_block_v0() const { + if (prune_state != prune_state_type::complete_legacy) + return {}; + + signed_block_v0 result(*static_cast(this)); + result.block_extensions = this->block_extensions; + + auto visitor = overloaded{ + [](const transaction_id_type &id) -> transaction_receipt_v0::trx_type { return id; }, + [](const packed_transaction &trx) -> transaction_receipt_v0::trx_type { + const auto& legacy = std::get(trx.get_prunable_data().prunable_data); + return packed_transaction_v0(trx.get_packed_transaction(), legacy.signatures, legacy.packed_context_free_data, trx.get_compression()); + }}; + + for (const transaction_receipt &r : transactions){ + result.transactions.emplace_back(transaction_receipt_v0{r, std::visit(visitor, r.trx)}); + } + return result; + } +} } /// namespace eosio::chain diff --git a/libraries/chain/block_header.cpp b/libraries/chain/block_header.cpp index 0efc7728a1f..eef0f5bee37 100644 --- a/libraries/chain/block_header.cpp +++ b/libraries/chain/block_header.cpp @@ -15,7 +15,7 @@ namespace eosio { namespace chain { return fc::endian_reverse_u32(id._hash[0]); } - block_id_type block_header::id()const + block_id_type block_header::calculate_id()const { // Do not include signed_block_header attributes in id, specifically exclude producer_signature. block_id_type result = digest(); //fc::sha256::hash(*static_cast(this)); diff --git a/libraries/chain/block_header_state.cpp b/libraries/chain/block_header_state.cpp index bdf2a4cc386..b7cd3036ac3 100644 --- a/libraries/chain/block_header_state.cpp +++ b/libraries/chain/block_header_state.cpp @@ -173,7 +173,7 @@ namespace eosio { namespace chain { signed_block_header pending_block_header_state::make_block_header( const checksum256_type& transaction_mroot, const checksum256_type& action_mroot, - const optional& new_producers, + const std::optional& new_producers, vector&& new_protocol_feature_activations, const protocol_feature_set& pfs )const @@ -208,10 +208,11 @@ namespace eosio { namespace chain { legacy::producer_schedule_type downgraded_producers; downgraded_producers.version = new_producers->version; for (const auto &p : new_producers->producers) { - p.authority.visit([&downgraded_producers, &p](const auto& auth){ + std::visit([&downgraded_producers, &p](const auto& auth) + { EOS_ASSERT(auth.keys.size() == 1 && auth.keys.front().weight == auth.threshold, producer_schedule_exception, "multisig block signing present before enabled!"); downgraded_producers.producers.emplace_back(legacy::producer_key{p.producer_name, auth.keys.front().key}); - }); + }, p.authority); } h.new_producers = std::move(downgraded_producers); } @@ -263,7 +264,7 @@ namespace eosio { namespace chain { EOS_ASSERT(wtmsig_enabled, producer_schedule_exception, "Block header producer_schedule_change_extension before activation of WTMsig Block Signatures" ); EOS_ASSERT( !was_pending_promoted, producer_schedule_exception, "cannot set pending producer schedule in the same block in which pending was promoted to active" ); - const auto& new_producer_schedule = exts.lower_bound(producer_schedule_change_extension::extension_id())->second.get(); + const auto& new_producer_schedule = std::get(exts.lower_bound(producer_schedule_change_extension::extension_id())->second); EOS_ASSERT( new_producer_schedule.version == active_schedule.version + 1, producer_schedule_exception, "wrong producer schedule version specified" ); EOS_ASSERT( prev_pending_schedule.schedule.producers.empty(), producer_schedule_exception, @@ -276,7 +277,7 @@ namespace eosio { namespace chain { protocol_feature_activation_set_ptr new_activated_protocol_features; { // handle protocol_feature_activation if( exts.count(protocol_feature_activation::extension_id() > 0) ) { - const auto& new_protocol_features = exts.lower_bound(protocol_feature_activation::extension_id())->second.get().protocol_features; + const auto& new_protocol_features = std::get(exts.lower_bound(protocol_feature_activation::extension_id())->second).protocol_features; validator( timestamp, prev_activated_protocol_features->protocol_features, new_protocol_features ); new_activated_protocol_features = std::make_shared( @@ -292,7 +293,7 @@ namespace eosio { namespace chain { block_header_state result( std::move( *static_cast(this) ) ); - result.id = h.id(); + result.id = h.calculate_id(); result.header = h; result.header_exts = std::move(exts); @@ -409,7 +410,7 @@ namespace eosio { namespace chain { void block_header_state::verify_signee( )const { - size_t num_keys_in_authority = valid_block_signing_authority.visit([](const auto &a){ return a.keys.size(); }); + auto num_keys_in_authority = std::visit([](const auto &a){ return a.keys.size(); }, valid_block_signing_authority); EOS_ASSERT(1 + additional_signatures.size() <= num_keys_in_authority, wrong_signing_key, "number of block signatures (${num_block_signatures}) exceeds number of keys in block signing authority (${num_keys})", ("num_block_signatures", 1 + additional_signatures.size()) @@ -449,7 +450,7 @@ namespace eosio { namespace chain { if( header_exts.count(protocol_feature_activation::extension_id()) == 0 ) return no_activations; - return header_exts.lower_bound(protocol_feature_activation::extension_id())->second.get().protocol_features; + return std::get(header_exts.lower_bound(protocol_feature_activation::extension_id())->second).protocol_features; } block_header_state::block_header_state( legacy::snapshot_block_header_state_v2&& snapshot ) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 822de1ea8b8..d647dc172d1 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -1,29 +1,17 @@ #include #include -#include +#include +#include +#include +#include +#include #include -#include #include - - -#define LOG_READ (std::ios::in | std::ios::binary) -#define LOG_WRITE (std::ios::out | std::ios::binary | std::ios::app) -#define LOG_RW ( std::ios::in | std::ios::out | std::ios::binary ) -#define LOG_WRITE_C "ab+" -#define LOG_RW_C "rb+" - -#ifndef _WIN32 -#define FC_FOPEN(p, m) fopen(p, m) -#else -#define FC_CAT(s1, s2) s1 ## s2 -#define FC_PREL(s) FC_CAT(L, s) -#define FC_FOPEN(p, m) _wfopen(p, FC_PREL(m)) -#endif +#include +#include namespace eosio { namespace chain { - const uint32_t block_log::min_supported_version = 1; - /** * History: * Version 1: complete block log from genesis @@ -31,184 +19,571 @@ namespace eosio { namespace chain { * this is in the form of an first_block_num that is written immediately after the version * Version 3: improvement on version 2 to not require the genesis state be provided when not starting * from block 1 + * Version 4: changes the block entry from the serialization of signed_block to a tuple of offset to next entry, + * compression_status and pruned_block. */ - const uint32_t block_log::max_supported_version = 3; - namespace detail { - using unique_file = std::unique_ptr; + enum versions { + initial_version = 1, + block_x_start_version = 2, + genesis_state_or_chain_id_version = 3, + pruned_transaction_version = 4 + }; + + const uint32_t block_log::min_supported_version = initial_version; + const uint32_t block_log::max_supported_version = pruned_transaction_version; + + struct block_log_preamble { + uint32_t version = 0; + uint32_t first_block_num = 0; + std::variant chain_context; + + chain_id_type chain_id() const { + return std::visit(overloaded{[](const chain_id_type& id) { return id; }, + [](const genesis_state& state) { return state.compute_chain_id(); }}, + chain_context); + } - class block_log_impl { - public: - signed_block_ptr head; - block_id_type head_id; - fc::cfile block_file; - fc::cfile index_file; - bool open_files = false; - bool genesis_written_to_block_log = false; - uint32_t version = 0; - uint32_t first_block_num = 0; - - inline void check_open_files() { - if( !open_files ) { - reopen(); - } - } - void reopen(); - - void close() { - if( block_file.is_open() ) - block_file.close(); - if( index_file.is_open() ) - index_file.close(); - open_files = false; - } + constexpr static int nbytes_with_chain_id = // the bytes count when the preamble contains chain_id + sizeof(version) + sizeof(first_block_num) + sizeof(chain_id_type) + sizeof(block_log::npos); + + void read_from(fc::datastream& ds, const fc::path& log_path) { + ds.read((char*)&version, sizeof(version)); + EOS_ASSERT(version > 0, block_log_exception, "Block log was not setup properly"); + EOS_ASSERT( + block_log::is_supported_version(version), block_log_unsupported_version, + "Unsupported version of block log. Block log version is ${version} while code supports version(s) " + "[${min},${max}], log file: ${log}", + ("version", version)("min", block_log::min_supported_version)("max", block_log::max_supported_version)("log", log_path.generic_string())); + + first_block_num = 1; + if (version != initial_version) { + ds.read((char*)&first_block_num, sizeof(first_block_num)); + } - template - void reset( const T& t, const signed_block_ptr& genesis_block, uint32_t first_block_num ); + if (block_log::contains_genesis_state(version, first_block_num)) { + chain_context.emplace(); + fc::raw::unpack(ds, std::get(chain_context)); + } else if (block_log::contains_chain_id(version, first_block_num)) { + chain_context = chain_id_type{}; + ds >> std::get(chain_context); + } else { + EOS_THROW(block_log_exception, + "Block log is not supported. version: ${ver} and first_block_num: ${fbn} does not contain " + "a genesis_state nor a chain_id.", + ("ver", version)("fbn", first_block_num)); + } + + if (version != initial_version) { + auto expected_totem = block_log::npos; + std::decay_t actual_totem; + ds.read((char*)&actual_totem, sizeof(actual_totem)); + + EOS_ASSERT( + actual_totem == expected_totem, block_log_exception, + "Expected separator between block log header and blocks was not found( expected: ${e}, actual: ${a} )", + ("e", fc::to_hex((char*)&expected_totem, sizeof(expected_totem)))( + "a", fc::to_hex((char*)&actual_totem, sizeof(actual_totem)))); + } + } - void write( const genesis_state& gs ); + template + void write_to(Stream& ds) const { + ds.write(reinterpret_cast(&version), sizeof(version)); + if (version != initial_version) { + ds.write(reinterpret_cast(&first_block_num), sizeof(first_block_num)); + + std::visit(overloaded{[&ds](const chain_id_type& id) { ds << id; }, + [&ds](const genesis_state& state) { + auto data = fc::raw::pack(state); + ds.write(data.data(), data.size()); + }}, + chain_context); + + auto totem = block_log::npos; + ds.write(reinterpret_cast(&totem), sizeof(totem)); + } + else { + const auto& state = std::get(chain_context); + auto data = fc::raw::pack(state); + ds.write(data.data(), data.size()); + } + } + }; + + struct log_entry_v4 { + // In version 4 of the irreversible blocks log format, these log entries consists of the following in order: + // 1. An uint32_t size for number of bytes from the start of this log entry to the start of the next log entry. + // 2. An uint8_t indicating the compression status for the serialization of the pruned_block following this. + // 3. The serialization of a signed_block representation of the block for the entry including padding. + + struct metadata_type { + packed_transaction::cf_compression_type compression = packed_transaction::cf_compression_type::none; + uint32_t size = 0; // the size of the log entry + }; + + metadata_type meta; + signed_block block; + }; + + namespace { + + template + T read_buffer(const char* buf) { + T result; + memcpy(&result, buf, sizeof(T)); + return result; + } - void write( const chain_id_type& chain_id ); + /// calculate the offset from the start of serialized block entry to block start + constexpr int offset_to_block_start(uint32_t version) { + return version >= pruned_transaction_version ? sizeof(uint32_t) + 1 : 0; + } - void flush(); + template + log_entry_v4::metadata_type unpack(Stream& ds, signed_block& block){ + log_entry_v4::metadata_type meta; + const auto start_pos = ds.tellp(); + fc::raw::unpack(ds, meta.size); + uint8_t compression; + fc::raw::unpack(ds, compression); + EOS_ASSERT(compression < static_cast(packed_transaction::cf_compression_type::COMPRESSION_TYPE_COUNT), block_log_exception, + "Unknown compression_type"); + meta.compression = static_cast(compression); + EOS_ASSERT(meta.compression == packed_transaction::cf_compression_type::none, block_log_exception, + "Only support compression_type none"); + block.unpack(ds, meta.compression); + const uint64_t current_stream_offset = ds.tellp() - start_pos; + // For a block which contains CFD (context free data) and the CFD is pruned afterwards, the entry.size may + // be the size before the CFD has been pruned while the actual serialized block does not have the CFD anymore. + // In this case, the serialized block has fewer bytes than what's indicated by entry.size. We need to + // skip over the extra bytes to allow ds to position to the last 8 bytes of the entry. + const int64_t bytes_to_skip = static_cast(meta.size) - sizeof(uint64_t) - current_stream_offset; + EOS_ASSERT(bytes_to_skip >= 0, block_log_exception, + "Invalid block log entry size"); + ds.skip(bytes_to_skip); + return meta; + } - uint64_t append(const signed_block_ptr& b); + template + void unpack(Stream& ds, log_entry_v4& entry){ + entry.meta = unpack(ds, entry.block); + } - template - static fc::optional extract_chain_context( const fc::path& data_dir, Lambda&& lambda ); - }; + std::vector pack(const signed_block& block, packed_transaction::cf_compression_type compression) { + const std::size_t padded_size = block.maximum_pruned_pack_size(compression); + static_assert( block_log::max_supported_version == pruned_transaction_version, + "Code was written to support format of version 4, need to update this code for latest format." ); + std::vector buffer(padded_size + offset_to_block_start(block_log::max_supported_version)); + fc::datastream stream(buffer.data(), buffer.size()); + + const uint32_t size = buffer.size() + sizeof(uint64_t); + stream.write((char*)&size, sizeof(size)); + fc::raw::pack(stream, static_cast(compression)); + block.pack(stream, compression); + return buffer; + } - void detail::block_log_impl::reopen() { - close(); + using log_entry = std::variant; - // open to create files if they don't exist - //ilog("Opening block log at ${path}", ("path", my->block_file.generic_string())); - block_file.open( LOG_WRITE_C ); - index_file.open( LOG_WRITE_C ); + const block_header& get_block_header(const log_entry& entry) { + return std::visit(overloaded{[](const signed_block_v0& v) -> const block_header& { return v; }, + [](const log_entry_v4& v) -> const block_header& { return v.block; }}, + entry); + } - close(); + template + void unpack(Stream& ds, log_entry& entry) { + std::visit( + overloaded{[&ds](signed_block_v0& v) { fc::raw::unpack(ds, v); }, + [&ds](log_entry_v4& v) { unpack(ds, v); }}, + entry); + } - block_file.open( LOG_RW_C ); - index_file.open( LOG_RW_C ); + void create_mapped_file(boost::iostreams::mapped_file_sink& sink, const std::string& path, uint64_t size) { + using namespace boost::iostreams; + mapped_file_params params(path); + params.flags = mapped_file::readwrite; + params.new_file_size = size; + params.length = size; + params.offset = 0; + sink.open(params); + } - open_files = true; + class index_writer { + public: + index_writer(const fc::path& block_index_name, uint32_t blocks_expected) + : current_offset(blocks_expected * sizeof(uint64_t)) { + create_mapped_file(index, block_index_name.generic_string(), current_offset); + } + void write(uint64_t pos) { + current_offset -= sizeof(pos); + memcpy(index.data() + current_offset, &pos, sizeof(pos)); } - class reverse_iterator { - public: - reverse_iterator(); - // open a block log file and return the total number of blocks in it - uint32_t open(const fc::path& block_file_name); - uint64_t previous(); - uint32_t version() const { return _version; } - uint32_t first_block_num() const { return _first_block_num; } - constexpr static uint32_t _buf_len = 1U << 24; - private: - void update_buffer(); - - unique_file _file; - uint32_t _version = 0; - uint32_t _first_block_num = 0; - uint32_t _last_block_num = 0; - uint32_t _blocks_found = 0; - uint32_t _blocks_expected = 0; - uint64_t _current_position_in_file = 0; - uint64_t _eof_position_in_file = 0; - uint64_t _end_of_buffer_position = _unset_position; - uint64_t _start_of_buffer_position = 0; - std::unique_ptr _buffer_ptr; - std::string _block_file_name; - constexpr static int64_t _unset_position = -1; - constexpr static uint64_t _position_size = sizeof(_current_position_in_file); - }; + void close() { index.close(); } - constexpr uint64_t buffer_location_to_file_location(uint32_t buffer_location) { return buffer_location << 3; } - constexpr uint32_t file_location_to_buffer_location(uint32_t file_location) { return file_location >> 3; } - - class index_writer { - public: - index_writer(const fc::path& block_index_name, uint32_t blocks_expected); - void write(uint64_t pos); - void complete(); - void update_buffer_position(); - constexpr static uint64_t _buffer_bytes = 1U << 22; - private: - void prepare_buffer(); - bool shift_buffer(); - - unique_file _file; - const std::string _block_index_name; - const uint32_t _blocks_expected; - uint32_t _block_written; - std::unique_ptr _buffer_ptr; - int64_t _current_position = 0; - int64_t _start_of_buffer_position = 0; - int64_t _end_of_buffer_position = 0; - constexpr static uint64_t _max_buffer_length = file_location_to_buffer_location(_buffer_bytes); - }; + private: + std::ptrdiff_t current_offset = 0; + boost::iostreams::mapped_file_sink index; + }; - /* - * @brief datastream adapter that adapts FILE* for use with fc unpack - * - * This class supports unpack functionality but not pack. - */ - class fileptr_datastream { - public: - explicit fileptr_datastream( FILE* file, const std::string& filename ) : _file(file), _filename(filename) {} - - void skip( size_t s ) { - auto status = fseek(_file, s, SEEK_CUR); - EOS_ASSERT( status == 0, block_log_exception, - "Could not seek past ${bytes} bytes in Block log file at '${blocks_log}'. Returned status: ${status}", - ("bytes", s)("blocks_log", _filename)("status", status) ); + struct bad_block_exception { + std::exception_ptr inner; + }; + + template + std::unique_ptr read_block(Stream&& ds, uint32_t version, uint32_t expect_block_num = 0) { + std::unique_ptr block; + if (version >= pruned_transaction_version) { + block = std::make_unique(); + unpack(ds, *block); + } else { + signed_block_v0 block_v0; + fc::raw::unpack(ds, block_v0); + block = std::make_unique(std::move(block_v0), true); + } + + if (expect_block_num != 0) { + EOS_ASSERT(!!block && block->block_num() == expect_block_num, block_log_exception, + "Wrong block was read from block log."); + } + + return block; + } + + template + block_id_type read_block_id(Stream&& ds, uint32_t version, uint32_t expect_block_num) { + if (version >= pruned_transaction_version) { + uint32_t size; + uint8_t compression; + fc::raw::unpack(ds, size); + fc::raw::unpack(ds, compression); + EOS_ASSERT(compression == static_cast(packed_transaction::cf_compression_type::none), + block_log_exception, "Only \"none\" compression type is supported."); + } + block_header bh; + fc::raw::unpack(ds, bh); + + EOS_ASSERT(bh.block_num() == expect_block_num, block_log_exception, + "Wrong block header was read from block log.", + ("returned", bh.block_num())("expected", expect_block_num)); + + return bh.calculate_id(); + } + + /// Provide the memory mapped view of the blocks.log file + class block_log_data : public chain::log_data_base { + block_log_preamble preamble; + uint64_t first_block_pos = block_log::npos; + public: + + block_log_data() = default; + block_log_data(const fc::path& path, mapmode mode = mapmode::readonly) { open(path, mode); } + + const block_log_preamble& get_preamble() const { return preamble; } + + fc::datastream open(const fc::path& path, mapmode mode = mapmode::readonly) { + if (file.is_open()) + file.close(); + file.open(path.string(), mode); + fc::datastream ds(this->data(), this->size()); + preamble.read_from(ds, path); + first_block_pos = ds.tellp(); + return ds; + } + + uint32_t version() const { return preamble.version; } + uint32_t first_block_num() const { return preamble.first_block_num; } + uint64_t first_block_position() const { return first_block_pos; } + chain_id_type chain_id() const { return preamble.chain_id(); } + + std::optional get_genesis_state() const { + return std::visit(overloaded{[](const chain_id_type&) { return std::optional{}; }, + [](const genesis_state& state) { return std::optional{state}; }}, + preamble.chain_context); + } + + uint32_t block_num_at(uint64_t position) const { + // to derive blknum_offset==14 see block_header.hpp and note on disk struct is packed + // block_timestamp_type timestamp; //bytes 0:3 + // account_name producer; //bytes 4:11 + // uint16_t confirmed; //bytes 12:13 + // block_id_type previous; //bytes 14:45, low 4 bytes is big endian block number of + // previous block + + EOS_ASSERT(position <= size(), block_log_exception, "Invalid block position ${position}", ("position", position)); + + int blknum_offset = 14; + blknum_offset += offset_to_block_start(version()); + uint32_t prev_block_num = read_buffer(data() + position + blknum_offset); + return fc::endian_reverse_u32(prev_block_num) + 1; + } + + std::pair,uint32_t> ro_stream_at(uint64_t pos) { + return std::make_pair(fc::datastream(file.const_data() + pos, file.size() - pos), version()); + } + + std::pair,uint32_t> rw_stream_at(uint64_t pos) { + return std::make_pair(fc::datastream(file.data() + pos, file.size() - pos), version()); + } + + /** + * Validate a block log entry WITHOUT deserializing the entire block data. + **/ + void light_validate_block_entry_at(uint64_t pos, uint32_t expected_block_num) const { + const uint32_t actual_block_num = block_num_at(pos); + + EOS_ASSERT(actual_block_num == expected_block_num, block_log_exception, + "At position ${pos} expected to find block number ${exp_bnum} but found ${act_bnum}", + ("pos", pos)("exp_bnum", expected_block_num)("act_bnum", actual_block_num)); + + if (version() >= pruned_transaction_version) { + uint32_t entry_size = read_buffer(data()+pos); + uint64_t entry_position = read_buffer(data() + pos + entry_size - sizeof(uint64_t)); + EOS_ASSERT(pos == entry_position, block_log_exception, + "The last 8 bytes in the block entry of block number ${n} does not contain its own position", ("n", actual_block_num)); } + } + + /** + * Validate a block log entry by deserializing the entire block data. + * + * @returns The tuple of block number and block id in the entry + **/ + static std::tuple + full_validate_block_entry(fc::datastream& ds, uint32_t previous_block_num, const block_id_type& previous_block_id, log_entry& entry) { + uint64_t pos = ds.tellp(); - bool read( char* d, size_t s ) { - size_t result = fread( d, 1, s, _file ); - EOS_ASSERT( result == s, block_log_exception, - "only able to read ${act} bytes of the expected ${exp} bytes in file: ${file}", - ("act",result)("exp",s)("file", _filename) ); - return true; + try { + unpack(ds, entry); + } catch (...) { + throw bad_block_exception{std::current_exception()}; } - bool get( unsigned char& c ) { return get( *(char*)&c ); } + const block_header& header = get_block_header(entry); - bool get( char& c ) { return read(&c, 1); } + auto id = header.calculate_id(); + auto block_num = block_header::num_from_id(id); - private: - FILE* const _file; - const std::string _filename; - }; - } + if (block_num != previous_block_num + 1) { + elog( "Block ${num} (${id}) skips blocks. Previous block in block log is block ${prev_num} (${previous})", + ("num", block_num)("id", id) + ("prev_num", previous_block_num)("previous", previous_block_id) ); + } + + if (previous_block_id != block_id_type() && previous_block_id != header.previous) { + elog("Block ${num} (${id}) does not link back to previous block. " + "Expected previous: ${expected}. Actual previous: ${actual}.", + ("num", block_num)("id", id)("expected", previous_block_id)("actual", header.previous)); + } + + uint64_t tmp_pos = std::numeric_limits::max(); + if (ds.remaining() >= sizeof(tmp_pos)) { + ds.read(reinterpret_cast(&tmp_pos), sizeof(tmp_pos)); + } + + EOS_ASSERT(pos == tmp_pos, block_log_exception, "the block position for block ${num} at the end of a block entry is incorrect", ("num", block_num)); + return std::make_tuple(block_num, id); + } + + void construct_index(const fc::path& index_file_name); + }; + + using block_log_index = eosio::chain::log_index; + + /// Provide the read only view for both blocks.log and blocks.index files + struct block_log_bundle { + fc::path block_file_name, index_file_name; // full pathname for blocks.log and blocks.index + block_log_data log_data; + block_log_index log_index; + + block_log_bundle(fc::path block_dir) { + block_file_name = block_dir / "blocks.log"; + index_file_name = block_dir / "blocks.index"; + + log_data.open(block_file_name); + log_index.open(index_file_name); + + uint32_t log_num_blocks = log_data.num_blocks(); + uint32_t index_num_blocks = log_index.num_blocks(); + + EOS_ASSERT( + log_num_blocks == index_num_blocks, block_log_exception, + "${block_file_name} says it has ${log_num_blocks} blocks which disagrees with ${index_num_blocks} indicated by ${index_file_name}", + ("block_file_name", block_file_name)("log_num_blocks", log_num_blocks)("index_num_blocks", index_num_blocks)("index_file_name", index_file_name)); + } + }; + + /// Used to traverse the block position (i.e. the last 8 bytes in each block log entry) of the blocks.log file + template + struct reverse_block_position_iterator { + const T& data; + uint64_t begin_position; + uint64_t current_position; + reverse_block_position_iterator(const T& data, uint64_t first_block_pos) + : data(data) + , begin_position(first_block_pos - sizeof(uint64_t)) + , current_position(data.size() - sizeof(uint64_t)) {} + + auto addr() const { return data.data() + current_position; } + + uint64_t get_value() { + if (current_position <= begin_position) + return block_log::npos; + return read_buffer(addr()); + } + + void set_value(uint64_t pos) { memcpy(addr(), &pos, sizeof(pos)); } + + reverse_block_position_iterator& operator++() { + EOS_ASSERT(current_position > begin_position && current_position < data.size(), block_log_exception, + "Block log file formatting is incorrect, it contains a block position value: ${pos}, which is not " + "in the range of (${begin_pos},${last_pos})", + ("pos", current_position)("begin_pos", begin_position)("last_pos", data.size())); - block_log::block_log(const fc::path& data_dir) - :my(new detail::block_log_impl()) { - open(data_dir); + current_position = read_buffer(addr()) - sizeof(uint64_t); + return *this; + } + }; + + template + reverse_block_position_iterator make_reverse_block_position_iterator(const BlockLogData& t) { + return reverse_block_position_iterator(t, t.first_block_position()); } - block_log::block_log(block_log&& other) { - my = std::move(other.my); + template + reverse_block_position_iterator make_reverse_block_position_iterator(const BlockLogData& t, + uint64_t first_block_position) { + return reverse_block_position_iterator(t, first_block_position); } - block_log::~block_log() { - if (my) { - flush(); - my->close(); - my.reset(); + void block_log_data::construct_index(const fc::path& index_file_path) { + std::string index_file_name = index_file_path.generic_string(); + ilog("Will write new blocks.index file ${file}", ("file", index_file_name)); + + const uint32_t num_blocks = this->num_blocks(); + + ilog("block log version= ${version}", ("version", this->version())); + + if (num_blocks == 0) { + return; } + + ilog("first block= ${first} last block= ${last}", + ("first", this->first_block_num())("last", (this->last_block_num()))); + + index_writer index(index_file_path, num_blocks); + uint32_t blocks_found = 0; + + for (auto iter = make_reverse_block_position_iterator(*this); + iter.get_value() != block_log::npos && blocks_found < num_blocks; ++iter, ++blocks_found) { + index.write(iter.get_value()); + } + + EOS_ASSERT(blocks_found == num_blocks, block_log_exception, + "Block log file at '${blocks_log}' formatting indicated last block: ${last_block_num}, first " + "block: ${first_block_num}, but found ${num} blocks", + ("blocks_log", index_file_name.replace(index_file_name.size() - 5, 5, "log"))( + "last_block_num", this->last_block_num())("first_block_num", + this->first_block_num())("num", blocks_found)); } - void block_log::open(const fc::path& data_dir) { - my->close(); + } // namespace + + struct block_log_verifier { + chain_id_type chain_id; + + void verify(const block_log_data& log, const boost::filesystem::path& log_path) { + if (chain_id.empty()) { + chain_id = log.chain_id(); + } else { + EOS_ASSERT(chain_id == log.chain_id(), block_log_exception, + "block log file ${path} has a different chain id", ("path", log_path.generic_string())); + } + } + }; + using block_log_catalog = eosio::chain::log_catalog; + + + namespace detail { + + /** + * The implementation detail for the read/write access to the block log/index + * + * @note All the non-static member functions require to fullfill the class invariant after execution unless + *exceptions are thrown. + * @invariant block_file.is_open() && index_file.is_open() + **/ + class block_log_impl { + public: + signed_block_ptr head; + block_log_catalog catalog; + fc::datastream block_file; + fc::datastream index_file; + bool genesis_written_to_block_log = false; + block_log_preamble preamble; + uint32_t future_version; + const size_t stride; + static uint32_t default_version; + + explicit block_log_impl(const block_log::config_type& config); + + static void ensure_file_exists(fc::cfile& f) { + if (fc::exists(f.get_file_path())) + return; + f.open(fc::cfile::create_or_update_rw_mode); + f.close(); + } + + uint64_t get_block_pos(uint32_t block_num); + + void reset(uint32_t first_block_num, std::variant&& chain_context); + + void flush(); - if (!fc::is_directory(data_dir)) - fc::create_directories(data_dir); + uint64_t append(const signed_block_ptr& b, packed_transaction::cf_compression_type segment_compression); + + // create futures for append, must call in order of blocks + std::future>> + create_append_future(boost::asio::io_context& thread_pool, + const signed_block_ptr& b, packed_transaction::cf_compression_type segment_compression); + uint64_t append(std::future>> f); + + uint64_t write_log_entry(const std::vector& block_buffer); + + void split_log(); + bool recover_from_incomplete_block_head(block_log_data& log_data, block_log_index& index); + + block_id_type read_block_id_by_num(uint32_t block_num); + std::unique_ptr read_block_by_num(uint32_t block_num); + void read_head(); + }; + uint32_t block_log_impl::default_version = block_log::max_supported_version; + } // namespace detail - my->block_file.set_file_path( data_dir / "blocks.log" ); - my->index_file.set_file_path( data_dir / "blocks.index" ); + block_log::block_log(const block_log::config_type& config) + : my(new detail::block_log_impl(config)) {} - my->reopen(); + block_log::~block_log() {} + + void block_log::set_version(uint32_t ver) { detail::block_log_impl::default_version = ver; } + uint32_t block_log::version() const { return my->preamble.version; } + + detail::block_log_impl::block_log_impl(const block_log::config_type& config) + : stride( config.stride ) + { + + if (!fc::is_directory(config.log_dir)) + fc::create_directories(config.log_dir); + + catalog.open(config.log_dir, config.retained_dir, config.archive_dir, "blocks"); + + catalog.max_retained_files = config.max_retained_files; + + block_file.set_file_path(config.log_dir / "blocks.log"); + index_file.set_file_path(config.log_dir / "blocks.index"); /* On startup of the block log, there are several states the log file and the index file can be * in relation to each other. * @@ -227,99 +602,172 @@ namespace eosio { namespace chain { * - If the index file head is not in the log file, delete the index and replay. * - If the index file head is in the log, but not up to date, replay from index head. */ - auto log_size = fc::file_size( my->block_file.get_file_path() ); - auto index_size = fc::file_size( my->index_file.get_file_path() ); + ensure_file_exists(block_file); + ensure_file_exists(index_file); + const auto log_size = fc::file_size(block_file.get_file_path()); + const auto index_size = fc::file_size(index_file.get_file_path()); if (log_size) { ilog("Log is nonempty"); - my->block_file.seek( 0 ); - my->version = 0; - my->block_file.read( (char*)&my->version, sizeof(my->version) ); - EOS_ASSERT( my->version > 0, block_log_exception, "Block log was not setup properly" ); - EOS_ASSERT( is_supported_version(my->version), block_log_unsupported_version, - "Unsupported version of block log. Block log version is ${version} while code supports version(s) [${min},${max}]", - ("version", my->version)("min", block_log::min_supported_version)("max", block_log::max_supported_version) ); - - - my->genesis_written_to_block_log = true; // Assume it was constructed properly. - if (my->version > 1){ - my->first_block_num = 0; - my->block_file.read( (char*)&my->first_block_num, sizeof(my->first_block_num) ); - EOS_ASSERT(my->first_block_num > 0, block_log_exception, "Block log is malformed, first recorded block number is 0 but must be greater than or equal to 1"); - } else { - my->first_block_num = 1; - } + block_log_data log_data(block_file.get_file_path()); + preamble = log_data.get_preamble(); + future_version = preamble.version; - my->head = read_head(); - if( my->head ) { - my->head_id = my->head->id(); - } else { - my->head_id = {}; - } + EOS_ASSERT(catalog.verifier.chain_id.empty() || catalog.verifier.chain_id == preamble.chain_id(), block_log_exception, + "block log file ${path} has a different chain id", ("path", block_file.get_file_path())); + + genesis_written_to_block_log = true; // Assume it was constructed properly. if (index_size) { ilog("Index is nonempty"); - uint64_t block_pos; - my->block_file.seek_end(-sizeof(uint64_t)); - my->block_file.read((char*)&block_pos, sizeof(block_pos)); - - uint64_t index_pos; - my->index_file.seek_end(-sizeof(uint64_t)); - my->index_file.read((char*)&index_pos, sizeof(index_pos)); - - if (block_pos < index_pos) { - ilog("block_pos < index_pos, close and reopen index_file"); - construct_index(); - } else if (block_pos > index_pos) { - ilog("Index is incomplete"); - construct_index(); + if (index_size % sizeof(uint64_t) == 0) { + block_log_index index(index_file.get_file_path()); + + if (log_data.last_block_position() != index.back()) { + if (!config.fix_irreversible_blocks) { + ilog("The last block positions from blocks.log and blocks.index are different, Reconstructing index..."); + log_data.construct_index(index_file.get_file_path()); + } + else if (!recover_from_incomplete_block_head(log_data, index)) { + block_log::repair_log(block_file.get_file_path().parent_path(), UINT32_MAX); + block_log::construct_index(block_file.get_file_path(), index_file.get_file_path()); + } + } else if (config.fix_irreversible_blocks) { + ilog("Irreversible blocks was not corrupted."); + } } + else { + if (config.fix_irreversible_blocks) { + block_log::repair_log(block_file.get_file_path().parent_path(), UINT32_MAX); + block_log::construct_index(block_file.get_file_path(), index_file.get_file_path()); + } + else { + log_data.construct_index(index_file.get_file_path()); + } + } } else { - ilog("Index is empty"); - construct_index(); + ilog("Index is empty. Reconstructing index..."); + log_data.construct_index(index_file.get_file_path()); } } else if (index_size) { - ilog("Index is nonempty, remove and recreate it"); - my->close(); - fc::remove_all( my->index_file.get_file_path() ); - my->reopen(); + ilog("Log file is empty while the index file is nonempty, discard the index file"); + boost::filesystem::resize_file(index_file.get_file_path(), 0); } + + block_file.open(fc::cfile::update_rw_mode); + index_file.open(fc::cfile::update_rw_mode); + if (log_size) + read_head(); } - uint64_t block_log::append(const signed_block_ptr& b) { - return my->append(b); + std::vector create_block_buffer( const signed_block& b, uint32_t version, packed_transaction::cf_compression_type segment_compression ) { + std::vector buffer; + + if (version >= pruned_transaction_version) { + buffer = pack(b, segment_compression); + } else { + auto block_ptr = b.to_signed_block_v0(); + EOS_ASSERT(block_ptr, block_log_append_fail, "Unable to convert block to legacy format"); + EOS_ASSERT(segment_compression == packed_transaction::cf_compression_type::none, block_log_append_fail, + "the compression must be \"none\" for legacy format"); + buffer = fc::raw::pack(*block_ptr); + } + + return buffer; } - uint64_t detail::block_log_impl::append(const signed_block_ptr& b) { + uint64_t detail::block_log_impl::write_log_entry(const std::vector& block_buffer) { + uint64_t pos = block_file.tellp(); + + block_file.write(block_buffer.data(), block_buffer.size()); + block_file.write((char*)&pos, sizeof(pos)); + index_file.write((char*)&pos, sizeof(pos)); + flush(); + return pos; + } + + uint64_t block_log::append(const signed_block_ptr& b, packed_transaction::cf_compression_type segment_compression) { + return my->append(b, segment_compression); + } + + uint64_t detail::block_log_impl::append(const signed_block_ptr& b, packed_transaction::cf_compression_type segment_compression) { try { EOS_ASSERT( genesis_written_to_block_log, block_log_append_fail, "Cannot append to block log until the genesis is first written" ); - check_open_files(); - block_file.seek_end(0); index_file.seek_end(0); - uint64_t pos = block_file.tellp(); - EOS_ASSERT(index_file.tellp() == sizeof(uint64_t) * (b->block_num() - first_block_num), + EOS_ASSERT(index_file.tellp() == sizeof(uint64_t) * (b->block_num() - preamble.first_block_num), block_log_append_fail, "Append to index file occuring at wrong position.", ("position", (uint64_t) index_file.tellp()) - ("expected", (b->block_num() - first_block_num) * sizeof(uint64_t))); - auto data = fc::raw::pack(*b); - block_file.write(data.data(), data.size()); - block_file.write((char*)&pos, sizeof(pos)); - index_file.write((char*)&pos, sizeof(pos)); - head = b; - head_id = b->id(); + ("expected", (b->block_num() - preamble.first_block_num) * sizeof(uint64_t))); - flush(); + std::vector buffer = create_block_buffer( *b, preamble.version, segment_compression ); + auto pos = write_log_entry(buffer); + head = b; + if (b->block_num() % stride == 0) { + split_log(); + } + return pos; + } + FC_LOG_AND_RETHROW() + } + uint64_t detail::block_log_impl::append(std::future>> f) { + try { + EOS_ASSERT( genesis_written_to_block_log, block_log_append_fail, "Cannot append to block log until the genesis is first written" ); + + block_file.seek_end(0); + index_file.seek_end(0); + auto[b, buffer] = f.get(); + EOS_ASSERT(index_file.tellp() == sizeof(uint64_t) * (b->block_num() - preamble.first_block_num), + block_log_append_fail, + "Append to index file occuring at wrong position.", + ("position", (uint64_t) index_file.tellp()) + ("expected", (b->block_num() - preamble.first_block_num) * sizeof(uint64_t))); + + auto pos = write_log_entry(buffer); + head = b; + if (b->block_num() % stride == 0) { + split_log(); + } return pos; } FC_LOG_AND_RETHROW() } - void block_log::flush() { - my->flush(); + std::future>> + block_log::create_append_future(boost::asio::io_context& thread_pool, const signed_block_ptr& b, packed_transaction::cf_compression_type segment_compression) { + return my->create_append_future(thread_pool, b, segment_compression); + } + + std::future>> + detail::block_log_impl::create_append_future(boost::asio::io_context& thread_pool, const signed_block_ptr& b, packed_transaction::cf_compression_type segment_compression) { + future_version = (b->block_num() % stride == 0) ? block_log::max_supported_version : future_version; + std::promise>> p; + std::future>> f = p.get_future(); + return async_thread_pool( thread_pool, [b, version=future_version, segment_compression]() { + return std::make_tuple(b, create_block_buffer(*b, version, segment_compression)); + } ); + } + + uint64_t block_log::append(std::future>> f) { + return my->append( std::move( f ) ); + } + + void detail::block_log_impl::split_log() { + block_file.close(); + index_file.close(); + + catalog.add(preamble.first_block_num, this->head->block_num(), block_file.get_file_path().parent_path(), "blocks"); + + block_file.open(fc::cfile::truncate_rw_mode); + index_file.open(fc::cfile::truncate_rw_mode); + preamble.version = block_log::max_supported_version; + preamble.chain_context = preamble.chain_id(); + preamble.first_block_num = this->head->block_num() + 1; + preamble.write_to(block_file); + flush(); } void detail::block_log_impl::flush() { @@ -327,629 +775,313 @@ namespace eosio { namespace chain { index_file.flush(); } - template - void detail::block_log_impl::reset( const T& t, const signed_block_ptr& first_block, uint32_t first_bnum ) { - close(); - - fc::remove_all( block_file.get_file_path() ); - fc::remove_all( index_file.get_file_path() ); - - reopen(); + void detail::block_log_impl::reset(uint32_t first_bnum, std::variant&& chain_context) { - version = 0; // version of 0 is invalid; it indicates that subsequent data was not properly written to the block log - first_block_num = first_bnum; + block_file.open(fc::cfile::truncate_rw_mode); + index_file.open(fc::cfile::truncate_rw_mode); - block_file.seek_end(0); - block_file.write((char*)&version, sizeof(version)); - block_file.write((char*)&first_block_num, sizeof(first_block_num)); + future_version = block_log_impl::default_version; + preamble.version = block_log_impl::default_version; + preamble.first_block_num = first_bnum; + preamble.chain_context = std::move(chain_context); + preamble.write_to(block_file); - write(t); + flush(); genesis_written_to_block_log = true; - - // append a totem to indicate the division between blocks and header - auto totem = block_log::npos; - block_file.write((char*)&totem, sizeof(totem)); - - if (first_block) { - append(first_block); - } else { - head.reset(); - head_id = {}; - } - - auto pos = block_file.tellp(); - static_assert( block_log::max_supported_version > 0, "a version number of zero is not supported" ); - - // going back to write correct version to indicate that all block log header data writes completed successfully - version = block_log::max_supported_version; - block_file.seek( 0 ); - block_file.write( (char*)&version, sizeof(version) ); - block_file.seek( pos ); - flush(); } - void block_log::reset( const genesis_state& gs, const signed_block_ptr& first_block ) { - my->reset(gs, first_block, 1); + void block_log::reset( const genesis_state& gs, const signed_block_ptr& first_block, packed_transaction::cf_compression_type segment_compression ) { + my->reset(1, gs); + append(first_block, segment_compression); } - void block_log::reset( const chain_id_type& chain_id, uint32_t first_block_num ) { - EOS_ASSERT( first_block_num > 1, block_log_exception, - "Block log version ${ver} needs to be created with a genesis state if starting from block number 1." ); - my->reset(chain_id, signed_block_ptr(), first_block_num); - } + void block_log::reset(const chain_id_type& chain_id, uint32_t first_block_num) { + EOS_ASSERT(first_block_num > 1, block_log_exception, + "Block log version ${ver} needs to be created with a genesis state if starting from block number 1."); - void detail::block_log_impl::write( const genesis_state& gs ) { - auto data = fc::raw::pack(gs); - block_file.write(data.data(), data.size()); - } + EOS_ASSERT(my->catalog.verifier.chain_id.empty() || chain_id == my->catalog.verifier.chain_id, block_log_exception, + "Trying to reset to the chain to a different chain id"); - void detail::block_log_impl::write( const chain_id_type& chain_id ) { - block_file << chain_id; + my->reset(first_block_num, chain_id); + my->head.reset(); } - signed_block_ptr block_log::read_block(uint64_t pos)const { - my->check_open_files(); - - my->block_file.seek(pos); - signed_block_ptr result = std::make_shared(); - auto ds = my->block_file.create_datastream(); - fc::raw::unpack(ds, *result); - return result; + std::unique_ptr detail::block_log_impl::read_block_by_num(uint32_t block_num) { + uint64_t pos = get_block_pos(block_num); + if (pos != block_log::npos) { + block_file.seek(pos); + return read_block(block_file, preamble.version, block_num); + } else { + auto [ds, version] = catalog.ro_stream_for_block(block_num); + if (ds.remaining()) + return read_block(ds, version, block_num); + } + return {}; } - void block_log::read_block_header(block_header& bh, uint64_t pos)const { - my->check_open_files(); - - my->block_file.seek(pos); - auto ds = my->block_file.create_datastream(); - fc::raw::unpack(ds, bh); + block_id_type detail::block_log_impl::read_block_id_by_num(uint32_t block_num) { + uint64_t pos = get_block_pos(block_num); + if (pos != block_log::npos) { + block_file.seek(pos); + return read_block_id(block_file, preamble.version, block_num); + } else { + auto [ds, version] = catalog.ro_stream_for_block(block_num); + if (ds.remaining()) + return read_block_id(ds, version, block_num); + } + return {}; } - signed_block_ptr block_log::read_block_by_num(uint32_t block_num)const { - try { - signed_block_ptr b; - uint64_t pos = get_block_pos(block_num); - if (pos != npos) { - b = read_block(pos); - EOS_ASSERT(b->block_num() == block_num, reversible_blocks_exception, - "Wrong block was read from block log.", ("returned", b->block_num())("expected", block_num)); - } - return b; - } FC_LOG_AND_RETHROW() + std::unique_ptr block_log::read_signed_block_by_num(uint32_t block_num) const { + return my->read_block_by_num(block_num); } - block_id_type block_log::read_block_id_by_num(uint32_t block_num)const { - try { - uint64_t pos = get_block_pos(block_num); - if (pos != npos) { - block_header bh; - read_block_header(bh, pos); - EOS_ASSERT(bh.block_num() == block_num, reversible_blocks_exception, - "Wrong block header was read from block log.", ("returned", bh.block_num())("expected", block_num)); - return bh.id(); - } - return {}; - } FC_LOG_AND_RETHROW() + block_id_type block_log::read_block_id_by_num(uint32_t block_num) const { + return my->read_block_id_by_num(block_num); } - uint64_t block_log::get_block_pos(uint32_t block_num) const { - my->check_open_files(); - if (!(my->head && block_num <= block_header::num_from_id(my->head_id) && block_num >= my->first_block_num)) - return npos; - my->index_file.seek(sizeof(uint64_t) * (block_num - my->first_block_num)); + uint64_t detail::block_log_impl::get_block_pos(uint32_t block_num) { + if (!(head && block_num <= head->block_num() && block_num >= preamble.first_block_num)) + return block_log::npos; + index_file.seek(sizeof(uint64_t) * (block_num - preamble.first_block_num)); uint64_t pos; - my->index_file.read((char*)&pos, sizeof(pos)); + index_file.read((char*)&pos, sizeof(pos)); return pos; } - signed_block_ptr block_log::read_head()const { - my->check_open_files(); - + void detail::block_log_impl::read_head() { uint64_t pos; - // Check that the file is not empty - my->block_file.seek_end(0); - if (my->block_file.tellp() <= sizeof(pos)) - return {}; - - my->block_file.seek_end(-sizeof(pos)); - my->block_file.read((char*)&pos, sizeof(pos)); - if (pos != npos) { - return read_block(pos); - } else { - return {}; + block_file.seek_end(-sizeof(pos)); + block_file.read((char*)&pos, sizeof(pos)); + if (pos != block_log::npos) { + block_file.seek(pos); + head = read_block(block_file, preamble.version); } } - const signed_block_ptr& block_log::head()const { + const signed_block_ptr& block_log::head() const { return my->head; } - const block_id_type& block_log::head_id()const { - return my->head_id; - } - uint32_t block_log::first_block_num() const { - return my->first_block_num; + if (!my->catalog.empty()) + return my->catalog.collection.begin()->first; + return my->preamble.first_block_num; } - void block_log::construct_index() { - ilog("Reconstructing Block Log Index..."); - my->close(); - - fc::remove_all( my->index_file.get_file_path() ); - - my->reopen(); - - - my->close(); - - block_log::construct_index(my->block_file.get_file_path(), my->index_file.get_file_path()); - - my->reopen(); - } // construct_index - void block_log::construct_index(const fc::path& block_file_name, const fc::path& index_file_name) { - detail::reverse_iterator block_log_iter; ilog("Will read existing blocks.log file ${file}", ("file", block_file_name.generic_string())); ilog("Will write new blocks.index file ${file}", ("file", index_file_name.generic_string())); - const uint32_t num_blocks = block_log_iter.open(block_file_name); + block_log_data log_data(block_file_name); + log_data.construct_index(index_file_name); + } - ilog("block log version= ${version}", ("version", block_log_iter.version())); + static void write_incomplete_block_data(const fc::path& blocks_dir, fc::time_point now, uint32_t block_num, const char* start, int size) { + auto tail_path = blocks_dir / std::string("blocks-bad-tail-").append(now).append(".log"); + fc::cfile tail; + tail.set_file_path(tail_path); + tail.open(fc::cfile::create_or_update_rw_mode); + tail.write(start, size); + + ilog("Data at tail end of block log which should contain the (incomplete) serialization of block ${num} " + "has been written out to '${tail_path}'.", + ("num", block_num + 1)("tail_path", tail_path)); + + } - if (num_blocks == 0) { - return; + bool detail::block_log_impl::recover_from_incomplete_block_head(block_log_data& log_data, block_log_index& index) { + const uint64_t pos = index.back(); + if (log_data.size() <= pos) { + // index refers to an invalid position, we cannot recover from it + return false; } - ilog("first block= ${first} last block= ${last}", - ("first", block_log_iter.first_block_num())("last", (block_log_iter.first_block_num() + num_blocks))); - - detail::index_writer index(index_file_name, num_blocks); - uint64_t position; - while ((position = block_log_iter.previous()) != npos) { - index.write(position); + log_entry entry; + if (preamble.version < pruned_transaction_version) { + entry.emplace(); } - index.complete(); + + const uint32_t expected_block_num = log_data.first_block_num() + index.num_blocks() - 1; + fc::datastream ds(log_data.data() + pos, log_data.size() - pos); + + try { + unpack(ds, entry); + const block_header& header = get_block_header(entry); + if (header.block_num() != expected_block_num) { + return false; + } + uint64_t tmp_pos = std::numeric_limits::max(); + ds.read(reinterpret_cast(&tmp_pos), sizeof(tmp_pos)); + if (tmp_pos != pos) + return false; + + const auto size_to_trim = ds.remaining(); + const auto trimmed_block_file_size = log_data.size() - size_to_trim; + + write_incomplete_block_data(block_file.get_file_path().parent_path(), fc::time_point::now(), expected_block_num + 1, + log_data.data() + trimmed_block_file_size, size_to_trim); + boost::filesystem::resize_file(block_file.get_file_path(), trimmed_block_file_size); + return true; + } catch (...) { return false; } } - fc::path block_log::repair_log( const fc::path& data_dir, uint32_t truncate_at_block ) { + fc::path block_log::repair_log(const fc::path& data_dir, uint32_t truncate_at_block, const char* reversible_block_dir_name) { ilog("Recovering Block Log..."); - EOS_ASSERT( fc::is_directory(data_dir) && fc::is_regular_file(data_dir / "blocks.log"), block_log_not_found, - "Block log not found in '${blocks_dir}'", ("blocks_dir", data_dir) ); + EOS_ASSERT(fc::is_directory(data_dir) && fc::is_regular_file(data_dir / "blocks.log"), block_log_not_found, + "Block log not found in '${blocks_dir}'", ("blocks_dir", data_dir)); + + if (truncate_at_block == 0) + truncate_at_block = UINT32_MAX; auto now = fc::time_point::now(); - auto blocks_dir = fc::canonical( data_dir ); - if( blocks_dir.filename().generic_string() == "." ) { - blocks_dir = blocks_dir.parent_path(); - } - auto backup_dir = blocks_dir.parent_path(); + auto blocks_dir = fc::canonical(data_dir); // canonical always returns an absolute path that has no symbolic link, dot, or dot-dot elements auto blocks_dir_name = blocks_dir.filename(); - EOS_ASSERT( blocks_dir_name.generic_string() != ".", block_log_exception, "Invalid path to blocks directory" ); - backup_dir = backup_dir / blocks_dir_name.generic_string().append("-").append( now ); + auto backup_dir = blocks_dir.parent_path() / blocks_dir_name.generic_string().append("-").append(now); - EOS_ASSERT( !fc::exists(backup_dir), block_log_backup_dir_exist, + EOS_ASSERT(!fc::exists(backup_dir), block_log_backup_dir_exist, "Cannot move existing blocks directory to already existing directory '${new_blocks_dir}'", - ("new_blocks_dir", backup_dir) ); - - fc::rename( blocks_dir, backup_dir ); - ilog( "Moved existing blocks directory to backup location: '${new_blocks_dir}'", ("new_blocks_dir", backup_dir) ); - - fc::create_directories(blocks_dir); - auto block_log_path = blocks_dir / "blocks.log"; - - ilog( "Reconstructing '${new_block_log}' from backed up block log", ("new_block_log", block_log_path) ); - - std::fstream old_block_stream; - std::fstream new_block_stream; - - old_block_stream.open( (backup_dir / "blocks.log").generic_string().c_str(), LOG_READ ); - new_block_stream.open( block_log_path.generic_string().c_str(), LOG_WRITE ); - - old_block_stream.seekg( 0, std::ios::end ); - uint64_t end_pos = old_block_stream.tellg(); - old_block_stream.seekg( 0 ); - - uint32_t version = 0; - old_block_stream.read( (char*)&version, sizeof(version) ); - EOS_ASSERT( version > 0, block_log_exception, "Block log was not setup properly" ); - EOS_ASSERT( is_supported_version(version), block_log_unsupported_version, - "Unsupported version of block log. Block log version is ${version} while code supports version(s) [${min},${max}]", - ("version", version)("min", block_log::min_supported_version)("max", block_log::max_supported_version) ); - - new_block_stream.write( (char*)&version, sizeof(version) ); + ("new_blocks_dir", backup_dir)); - uint32_t first_block_num = 1; - if (version != 1) { - old_block_stream.read ( (char*)&first_block_num, sizeof(first_block_num) ); - - // this assert is only here since repair_log is only used for --hard-replay-blockchain, which removes any - // existing state, if another API needs to use it, this can be removed and the check for the first block's - // previous block id will need to accommodate this. - EOS_ASSERT( first_block_num == 1, block_log_exception, - "Block log ${file} must contain a genesis state and start at block number 1. This block log " - "starts at block number ${first_block_num}.", - ("file", (backup_dir / "blocks.log").generic_string())("first_block_num", first_block_num)); - - new_block_stream.write( (char*)&first_block_num, sizeof(first_block_num) ); + fc::create_directories(backup_dir); + fc::rename(blocks_dir / "blocks.log", backup_dir / "blocks.log"); + if (fc::exists(blocks_dir/ "blocks.index")) { + fc::rename(blocks_dir/ "blocks.index", backup_dir/ "blocks.index"); } - - if (contains_genesis_state(version, first_block_num)) { - genesis_state gs; - fc::raw::unpack(old_block_stream, gs); - - auto data = fc::raw::pack( gs ); - new_block_stream.write( data.data(), data.size() ); + if (strlen(reversible_block_dir_name) && fc::is_directory(blocks_dir/reversible_block_dir_name)) { + fc::rename(blocks_dir/ reversible_block_dir_name, backup_dir/ reversible_block_dir_name); } - else if (contains_chain_id(version, first_block_num)) { - chain_id_type chain_id; - old_block_stream >> chain_id; + ilog("Moved existing blocks directory to backup location: '${new_blocks_dir}'", ("new_blocks_dir", backup_dir)); - new_block_stream << chain_id; - } - else { - EOS_THROW( block_log_exception, - "Block log ${file} is not supported. version: ${ver} and first_block_num: ${fbn} does not contain " - "a genesis_state nor a chain_id.", - ("file", (backup_dir / "blocks.log").generic_string())("ver", version)("fbn", first_block_num)); - } + const auto block_log_path = blocks_dir / "blocks.log"; + const auto block_file_name = block_log_path.generic_string(); - if (version != 1) { - auto expected_totem = npos; - std::decay_t actual_totem; - old_block_stream.read ( (char*)&actual_totem, sizeof(actual_totem) ); + ilog("Reconstructing '${new_block_log}' from backed up block log", ("new_block_log", block_file_name)); - EOS_ASSERT(actual_totem == expected_totem, block_log_exception, - "Expected separator between block log header and blocks was not found( expected: ${e}, actual: ${a} )", - ("e", fc::to_hex((char*)&expected_totem, sizeof(expected_totem) ))("a", fc::to_hex((char*)&actual_totem, sizeof(actual_totem) ))); + block_log_data log_data; + auto ds = log_data.open(backup_dir / "blocks.log"); + auto pos = ds.tellp(); + std::string error_msg; + uint32_t block_num = log_data.first_block_num() - 1; + block_id_type block_id; - new_block_stream.write( (char*)&actual_totem, sizeof(actual_totem) ); + log_entry entry; + if (log_data.version() < pruned_transaction_version) { + entry.emplace(); } - std::exception_ptr except_ptr; - vector incomplete_block_data; - optional bad_block; - uint32_t block_num = 0; - - block_id_type previous; - - uint64_t pos = old_block_stream.tellg(); - while( pos < end_pos ) { - signed_block tmp; - + try { try { - fc::raw::unpack(old_block_stream, tmp); - } catch( ... ) { - except_ptr = std::current_exception(); - incomplete_block_data.resize( end_pos - pos ); - old_block_stream.read( incomplete_block_data.data(), incomplete_block_data.size() ); - break; - } - - auto id = tmp.id(); - if( block_header::num_from_id(previous) + 1 != block_header::num_from_id(id) ) { - elog( "Block ${num} (${id}) skips blocks. Previous block in block log is block ${prev_num} (${previous})", - ("num", block_header::num_from_id(id))("id", id) - ("prev_num", block_header::num_from_id(previous))("previous", previous) ); - } - if( previous != tmp.previous ) { - elog( "Block ${num} (${id}) does not link back to previous block. " - "Expected previous: ${expected}. Actual previous: ${actual}.", - ("num", block_header::num_from_id(id))("id", id)("expected", previous)("actual", tmp.previous) ); - } - previous = id; - - uint64_t tmp_pos = std::numeric_limits::max(); - if( (static_cast(old_block_stream.tellg()) + sizeof(pos)) <= end_pos ) { - old_block_stream.read( reinterpret_cast(&tmp_pos), sizeof(tmp_pos) ); + while (ds.remaining() > 0 && block_num < truncate_at_block) { + std::tie(block_num, block_id) = block_log_data::full_validate_block_entry(ds, block_num, block_id, entry); + if (block_num % 1000 == 0) + ilog("Verified block ${num}", ("num", block_num)); + pos = ds.tellp(); + } } - if( pos != tmp_pos ) { - bad_block.emplace(std::move(tmp)); - break; + catch (const bad_block_exception& e) { + write_incomplete_block_data(blocks_dir, now, block_num, log_data.data() + pos, log_data.size() - pos); + std::rethrow_exception(e.inner); } - - auto data = fc::raw::pack(tmp); - new_block_stream.write( data.data(), data.size() ); - new_block_stream.write( reinterpret_cast(&pos), sizeof(pos) ); - block_num = tmp.block_num(); - if(block_num % 1000 == 0) - ilog( "Recovered block ${num}", ("num", block_num) ); - pos = new_block_stream.tellp(); - if( block_num == truncate_at_block ) - break; + } catch (const fc::exception& e) { + error_msg = e.what(); + } catch (const std::exception& e) { + error_msg = e.what(); + } catch (...) { + error_msg = "unrecognized exception"; } - if( bad_block.valid() ) { - ilog( "Recovered only up to block number ${num}. Last block in block log was not properly committed:\n${last_block}", - ("num", block_num)("last_block", *bad_block) ); - } else if( except_ptr ) { - std::string error_msg; - - try { - std::rethrow_exception(except_ptr); - } catch( const fc::exception& e ) { - error_msg = e.what(); - } catch( const std::exception& e ) { - error_msg = e.what(); - } catch( ... ) { - error_msg = "unrecognized exception"; - } - - ilog( "Recovered only up to block number ${num}. " - "The block ${next_num} could not be deserialized from the block log due to error:\n${error_msg}", - ("num", block_num)("next_num", block_num+1)("error_msg", error_msg) ); - - auto tail_path = blocks_dir / std::string("blocks-bad-tail-").append( now ).append(".log"); - if( !fc::exists(tail_path) && incomplete_block_data.size() > 0 ) { - std::fstream tail_stream; - tail_stream.open( tail_path.generic_string().c_str(), LOG_WRITE ); - tail_stream.write( incomplete_block_data.data(), incomplete_block_data.size() ); - - ilog( "Data at tail end of block log which should contain the (incomplete) serialization of block ${num} " - "has been written out to '${tail_path}'.", - ("num", block_num+1)("tail_path", tail_path) ); - } - } else if( block_num == truncate_at_block && pos < end_pos ) { - ilog( "Stopped recovery of block log early at specified block number: ${stop}.", ("stop", truncate_at_block) ); + fc::cfile new_block_file; + new_block_file.set_file_path(block_log_path); + new_block_file.open(fc::cfile::create_or_update_rw_mode); + new_block_file.write(log_data.data(), pos); + + if (error_msg.size()) { + ilog("Recovered only up to block number ${num}. " + "The block ${next_num} could not be deserialized from the block log due to error:\n${error_msg}", + ("num", block_num)("next_num", block_num + 1)("error_msg", error_msg)); + } else if (block_num == truncate_at_block && pos < log_data.size()) { + ilog("Stopped recovery of block log early at specified block number: ${stop}.", ("stop", truncate_at_block)); } else { - ilog( "Existing block log was undamaged. Recovered all irreversible blocks up to block number ${num}.", ("num", block_num) ); + ilog("Existing block log was undamaged. Recovered all irreversible blocks up to block number ${num}.", + ("num", block_num)); } - return backup_dir; } - template - fc::optional detail::block_log_impl::extract_chain_context( const fc::path& data_dir, Lambda&& lambda ) { - EOS_ASSERT( fc::is_directory(data_dir) && fc::is_regular_file(data_dir / "blocks.log"), block_log_not_found, - "Block log not found in '${blocks_dir}'", ("blocks_dir", data_dir) ); - - std::fstream block_stream; - block_stream.open( (data_dir / "blocks.log").generic_string().c_str(), LOG_READ ); - - uint32_t version = 0; - block_stream.read( (char*)&version, sizeof(version) ); - EOS_ASSERT( version >= block_log::min_supported_version && version <= block_log::max_supported_version, block_log_unsupported_version, - "Unsupported version of block log. Block log version is ${version} while code supports version(s) [${min},${max}]", - ("version", version)("min", block_log::min_supported_version)("max", block_log::max_supported_version) ); - - uint32_t first_block_num = 1; - if (version != 1) { - block_stream.read ( (char*)&first_block_num, sizeof(first_block_num) ); - } - - return lambda(block_stream, version, first_block_num); - } - - fc::optional block_log::extract_genesis_state( const fc::path& data_dir ) { - return detail::block_log_impl::extract_chain_context(data_dir, [](std::fstream& block_stream, uint32_t version, uint32_t first_block_num ) -> fc::optional { - if (contains_genesis_state(version, first_block_num)) { - genesis_state gs; - fc::raw::unpack(block_stream, gs); - return gs; - } - - // current versions only have a genesis state if they start with block number 1 - return fc::optional(); - }); + std::optional block_log::extract_genesis_state( const fc::path& block_dir ) { + boost::filesystem::path p(block_dir / "blocks.log"); + for_each_file_in_dir_matches(block_dir, R"(blocks-1-\d+\.log)", [&p](boost::filesystem::path log_path) { p = log_path; }); + return block_log_data(p).get_genesis_state(); } - + chain_id_type block_log::extract_chain_id( const fc::path& data_dir ) { - return *(detail::block_log_impl::extract_chain_context(data_dir, [](std::fstream& block_stream, uint32_t version, uint32_t first_block_num ) -> fc::optional { - // supported versions either contain a genesis state, or else the chain id only - if (contains_genesis_state(version, first_block_num)) { - genesis_state gs; - fc::raw::unpack(block_stream, gs); - return gs.compute_chain_id(); - } - EOS_ASSERT( contains_chain_id(version, first_block_num), block_log_exception, - "Block log error! version: ${version} with first_block_num: ${num} does not contain a " - "chain id or genesis state, so the chain id cannot be determined.", - ("version", version)("num", first_block_num) ); - chain_id_type chain_id; - fc::raw::unpack(block_stream, chain_id); - return chain_id; - })); - } - - detail::reverse_iterator::reverse_iterator() - : _file(nullptr, &fclose) - , _buffer_ptr(std::make_unique(_buf_len)) { - } - - uint32_t detail::reverse_iterator::open(const fc::path& block_file_name) { - _block_file_name = block_file_name.generic_string(); - _file.reset( FC_FOPEN(_block_file_name.c_str(), "r")); - EOS_ASSERT( _file, block_log_exception, "Could not open Block log file at '${blocks_log}'", ("blocks_log", _block_file_name) ); - _end_of_buffer_position = _unset_position; - - //read block log to see if version 1 or 2 and get first blocknum (implicit 1 if version 1) - _version = 0; - auto size = fread((char*)&_version, sizeof(_version), 1, _file.get()); - EOS_ASSERT( size == 1, block_log_exception, "Block log file at '${blocks_log}' could not be read.", ("file", _block_file_name) ); - EOS_ASSERT( block_log::is_supported_version(_version), block_log_unsupported_version, - "block log version ${v} is not supported", ("v", _version)); - if (_version == 1) { - _first_block_num = 1; - } - else { - size = fread((char*)&_first_block_num, sizeof(_first_block_num), 1, _file.get()); - EOS_ASSERT( size == 1, block_log_exception, "Block log file at '${blocks_log}' not formatted consistently with version ${v}.", ("file", _block_file_name)("v", _version) ); - } - - auto status = fseek(_file.get(), 0, SEEK_END); - EOS_ASSERT( status == 0, block_log_exception, "Could not open Block log file at '${blocks_log}'. Returned status: ${status}", ("blocks_log", _block_file_name)("status", status) ); - - _eof_position_in_file = ftell(_file.get()); - EOS_ASSERT( _eof_position_in_file > 0, block_log_exception, "Block log file at '${blocks_log}' could not be read.", ("blocks_log", _block_file_name) ); - _current_position_in_file = _eof_position_in_file - _position_size; - - update_buffer(); - - _blocks_found = 0; - char* buf = _buffer_ptr.get(); - const uint32_t index_of_pos = _current_position_in_file - _start_of_buffer_position; - const uint64_t block_pos = *reinterpret_cast(buf + index_of_pos); - - if (block_pos == block_log::npos) { - return 0; - } - - uint32_t bnum = 0; - if (block_pos >= _start_of_buffer_position) { - const uint32_t index_of_block = block_pos - _start_of_buffer_position; - bnum = *reinterpret_cast(buf + index_of_block + trim_data::blknum_offset); //block number of previous block (is big endian) - } - else { - const auto blknum_offset_pos = block_pos + trim_data::blknum_offset; - auto status = fseek(_file.get(), blknum_offset_pos, SEEK_SET); - EOS_ASSERT( status == 0, block_log_exception, "Could not seek in '${blocks_log}' to position: ${pos}. Returned status: ${status}", ("blocks_log", _block_file_name)("pos", blknum_offset_pos)("status", status) ); - auto size = fread((void*)&bnum, sizeof(bnum), 1, _file.get()); - EOS_ASSERT( size == 1, block_log_exception, "Could not read in '${blocks_log}' at position: ${pos}", ("blocks_log", _block_file_name)("pos", blknum_offset_pos) ); - } - _last_block_num = fc::endian_reverse_u32(bnum) + 1; //convert from big endian to little endian and add 1 - _blocks_expected = _last_block_num - _first_block_num + 1; - return _blocks_expected; + return block_log_data(data_dir / "blocks.log").chain_id(); } - uint64_t detail::reverse_iterator::previous() { - EOS_ASSERT( _current_position_in_file != block_log::npos, - block_log_exception, - "Block log file at '${blocks_log}' first block already returned by former call to previous(), it is no longer valid to call this function.", ("blocks_log", _block_file_name) ); - - if (_version == 1 && _blocks_found == _blocks_expected) { - _current_position_in_file = block_log::npos; - return _current_position_in_file; - } - - if (_start_of_buffer_position > _current_position_in_file) { - update_buffer(); - } - - char* buf = _buffer_ptr.get(); - auto offset = _current_position_in_file - _start_of_buffer_position; - uint64_t block_location_in_file = *reinterpret_cast(buf + offset); - - ++_blocks_found; - if (block_location_in_file == block_log::npos) { - _current_position_in_file = block_location_in_file; - EOS_ASSERT( _blocks_found != _blocks_expected, - block_log_exception, - "Block log file at '${blocks_log}' formatting indicated last block: ${last_block_num}, first block: ${first_block_num}, but found ${num} blocks", - ("blocks_log", _block_file_name)("last_block_num", _last_block_num)("first_block_num", _first_block_num)("num", _blocks_found) ); + size_t prune_trxs(fc::datastream strm, uint32_t block_num, std::vector& ids, uint32_t version) { + + EOS_ASSERT(version >= pruned_transaction_version, block_log_exception, + "The block log version ${version} does not support transaction pruning.", ("version", version)); + + auto read_strm = strm; + log_entry_v4 entry; + unpack(read_strm, entry); + + EOS_ASSERT(entry.block.block_num() == block_num, block_log_exception, + "Wrong block was read from block log."); + + auto pruner = overloaded{[](transaction_id_type&) { return false; }, + [&ids](packed_transaction& ptx) { + auto it = std::find(ids.begin(), ids.end(), ptx.id()); + if (it != ids.end()) { + ptx.prune_all(); + // remove the found entry from ids + ids.erase(it); + return true; + } + return false; + }}; + + size_t num_trx_pruned = 0; + for (auto& trx : entry.block.transactions) { + num_trx_pruned += std::visit(pruner, trx.trx); } - else { - const uint64_t previous_position_in_file = _current_position_in_file; - _current_position_in_file = block_location_in_file - _position_size; - EOS_ASSERT( _current_position_in_file < previous_position_in_file, - block_log_exception, - "Block log file at '${blocks_log}' formatting is incorrect, indicates position later location in file: ${pos}, which was retrieved at: ${orig_pos}.", - ("blocks_log", _block_file_name)("pos", _current_position_in_file)("orig_pos", previous_position_in_file) ); + if (num_trx_pruned > 0){ + entry.block.prune_state = signed_block::prune_state_type::incomplete; } - - return block_location_in_file; + strm.skip(offset_to_block_start(version)); + entry.block.pack(strm, entry.meta.compression); + return num_trx_pruned; } - void detail::reverse_iterator::update_buffer() { - EOS_ASSERT( _current_position_in_file != block_log::npos, block_log_exception, "Block log file not setup properly" ); + size_t block_log::prune_transactions(uint32_t block_num, std::vector& ids) { - // since we need to read in a new section, just need to ensure the next position is at the very end of the buffer - _end_of_buffer_position = _current_position_in_file + _position_size; - if (_end_of_buffer_position < _buf_len) { - _start_of_buffer_position = 0; - } - else { - _start_of_buffer_position = _end_of_buffer_position - _buf_len; + auto [strm, version] = my->catalog.rw_stream_for_block(block_num); + if (strm.remaining()) { + return prune_trxs(strm, block_num, ids, version); } - auto status = fseek(_file.get(), _start_of_buffer_position, SEEK_SET); - EOS_ASSERT( status == 0, block_log_exception, "Could not seek in '${blocks_log}' to position: ${pos}. Returned status: ${status}", ("blocks_log", _block_file_name)("pos", _start_of_buffer_position)("status", status) ); - char* buf = _buffer_ptr.get(); - auto size = fread((void*)buf, (_end_of_buffer_position - _start_of_buffer_position), 1, _file.get());//read tail of blocks.log file into buf - EOS_ASSERT( size == 1, block_log_exception, "blocks.log read fails" ); - } + const uint64_t pos = my->get_block_pos(block_num); + EOS_ASSERT(pos != npos, block_log_exception, "Specified block_num ${block_num} does not exist in block log.", + ("block_num", block_num)); - detail::index_writer::index_writer(const fc::path& block_index_name, uint32_t blocks_expected) - : _file(nullptr, &fclose) - , _block_index_name(block_index_name.generic_string()) - , _blocks_expected(blocks_expected) - , _block_written(blocks_expected) - , _buffer_ptr(std::make_unique(_max_buffer_length)) { - } - - void detail::index_writer::write(uint64_t pos) { - prepare_buffer(); - uint64_t* buffer = _buffer_ptr.get(); - buffer[_current_position - _start_of_buffer_position] = pos; - --_current_position; - if ((_block_written & 0xfffff) == 0) { //periodically print a progress indicator - ilog("block: ${block_written} position in file: ${pos}", ("block_written", _block_written)("pos",pos)); - } - --_block_written; - } - - void detail::index_writer::prepare_buffer() { - if (_file == nullptr) { - _file.reset(FC_FOPEN(_block_index_name.c_str(), "w")); - EOS_ASSERT( _file, block_log_exception, "Could not open Block index file at '${blocks_index}'", ("blocks_index", _block_index_name) ); - // allocate 8 bytes for each block position to store - const auto full_file_size = buffer_location_to_file_location(_blocks_expected); - auto status = fseek(_file.get(), full_file_size, SEEK_SET); - EOS_ASSERT( status == 0, block_log_exception, "Could not allocate in '${blocks_index}' storage for all the blocks, size: ${size}. Returned status: ${status}", ("blocks_index", _block_index_name)("size", full_file_size)("status", status) ); - const auto block_end = file_location_to_buffer_location(full_file_size); - _current_position = block_end - 1; - update_buffer_position(); - } - - shift_buffer(); - } - - bool detail::index_writer::shift_buffer() { - if (_current_position >= _start_of_buffer_position) { - return false; - } - - const auto file_location_start = buffer_location_to_file_location(_start_of_buffer_position); - - auto status = fseek(_file.get(), file_location_start, SEEK_SET); - EOS_ASSERT( status == 0, block_log_exception, "Could not navigate in '${blocks_index}' file_location_start: ${loc}, _start_of_buffer_position: ${_start_of_buffer_position}. Returned status: ${status}", ("blocks_index", _block_index_name)("loc", file_location_start)("_start_of_buffer_position",_start_of_buffer_position)("status", status) ); - - const auto buffer_size = _end_of_buffer_position - _start_of_buffer_position; - const auto file_size = buffer_location_to_file_location(buffer_size); - uint64_t* buf = _buffer_ptr.get(); - auto size = fwrite((void*)buf, file_size, 1, _file.get()); - EOS_ASSERT( size == 1, block_log_exception, "Writing Block Index file '${file}' failed at location: ${loc}", ("file", _block_index_name)("loc", file_location_start) ); - update_buffer_position(); - return true; - } - - void detail::index_writer::complete() { - const bool shifted = shift_buffer(); - EOS_ASSERT(shifted, block_log_exception, "Failed to write buffer to '${blocks_index}'", ("blocks_index", _block_index_name) ); - EOS_ASSERT(_current_position == -1, - block_log_exception, - "Should have written buffer, starting at the 0 index block position, to '${blocks_index}' but instead writing ${pos} position", - ("blocks_index", _block_index_name)("pos", _current_position) ); - } - - void detail::index_writer::update_buffer_position() { - _end_of_buffer_position = _current_position + 1; - if (_end_of_buffer_position < _max_buffer_length) { - _start_of_buffer_position = 0; - } - else { - _start_of_buffer_position = _end_of_buffer_position - _max_buffer_length; - } + using boost::iostreams::mapped_file_sink; + mapped_file_sink sink(my->block_file.get_file_path().string(), mapped_file_sink::max_length, 0); + fc::datastream ds(sink.data() + pos , sink.size() - pos); + return prune_trxs(ds, block_num, ids, my->preamble.version); } bool block_log::contains_genesis_state(uint32_t version, uint32_t first_block_num) { - return version <= 2 || first_block_num == 1; + return version < genesis_state_or_chain_id_version || first_block_num == 1; } bool block_log::contains_chain_id(uint32_t version, uint32_t first_block_num) { - return version >= 3 && first_block_num > 1; + return version >= genesis_state_or_chain_id_version && first_block_num > 1; } bool block_log::is_supported_version(uint32_t version) { @@ -957,231 +1089,116 @@ namespace eosio { namespace chain { } bool block_log::trim_blocklog_front(const fc::path& block_dir, const fc::path& temp_dir, uint32_t truncate_at_block) { - using namespace std; EOS_ASSERT( block_dir != temp_dir, block_log_exception, "block_dir and temp_dir need to be different directories" ); + ilog("In directory ${dir} will trim all blocks before block ${n} from blocks.log and blocks.index.", ("dir", block_dir.generic_string())("n", truncate_at_block)); - trim_data original_block_log(block_dir); - if (truncate_at_block <= original_block_log.first_block) { - ilog("There are no blocks before block ${n} so do nothing.", ("n", truncate_at_block)); + + block_log_bundle log_bundle(block_dir); + + if (truncate_at_block <= log_bundle.log_data.first_block_num()) { + dlog("There are no blocks before block ${n} so do nothing.", ("n", truncate_at_block)); return false; } - if (truncate_at_block > original_block_log.last_block) { - ilog("All blocks are before block ${n} so do nothing (trim front would delete entire blocks.log).", ("n", truncate_at_block)); + if (truncate_at_block > log_bundle.log_data.last_block_num()) { + dlog("All blocks are before block ${n} so do nothing (trim front would delete entire blocks.log).", ("n", truncate_at_block)); return false; } // ****** create the new block log file and write out the header for the file fc::create_directories(temp_dir); fc::path new_block_filename = temp_dir / "blocks.log"; - if (fc::remove(new_block_filename)) { - ilog("Removing old blocks.out file"); - } - fc::cfile new_block_file; - new_block_file.set_file_path(new_block_filename); - // need to open as append since the file doesn't already exist, then reopen without append to allow writing the - // file in any order - new_block_file.open( LOG_WRITE_C ); - new_block_file.close(); - new_block_file.open( LOG_RW_C ); - - static_assert( block_log::max_supported_version == 3, - "Code was written to support version 3 format, need to update this code for latest format." ); - uint32_t version = block_log::max_supported_version; - new_block_file.seek(0); - new_block_file.write((char*)&version, sizeof(version)); - new_block_file.write((char*)&truncate_at_block, sizeof(truncate_at_block)); - - new_block_file << original_block_log.chain_id; - - // append a totem to indicate the division between blocks and header - auto totem = block_log::npos; - new_block_file.write((char*)&totem, sizeof(totem)); - - const auto new_block_file_first_block_pos = new_block_file.tellp(); - // ****** end of new block log header - - // copy over remainder of block log to new block log - auto buffer = make_unique(detail::reverse_iterator::_buf_len); - char* buf = buffer.get(); - - // offset bytes to shift from old blocklog position to new blocklog position - const uint64_t original_file_block_pos = original_block_log.block_pos(truncate_at_block); - const uint64_t pos_delta = original_file_block_pos - new_block_file_first_block_pos; - auto status = fseek(original_block_log.blk_in, 0, SEEK_END); - EOS_ASSERT( status == 0, block_log_exception, "blocks.log seek failed" ); - - // all blocks to copy to the new blocklog - const uint64_t to_write = ftell(original_block_log.blk_in) - original_file_block_pos; - const auto pos_size = sizeof(uint64_t); - - // start with the last block's position stored at the end of the block - uint64_t original_pos = ftell(original_block_log.blk_in) - pos_size; - - const auto num_blocks = original_block_log.last_block - truncate_at_block + 1; + + static_assert( block_log::max_supported_version == pruned_transaction_version, + "Code was written to support format of version 4 or lower, need to update this code for latest format." ); + + const auto preamble_size = block_log_preamble::nbytes_with_chain_id; + const auto num_blocks_to_truncate = truncate_at_block - log_bundle.log_data.first_block_num(); + const uint64_t first_kept_block_pos = log_bundle.log_index.nth_block_position(num_blocks_to_truncate); + const uint64_t nbytes_to_trim = first_kept_block_pos - preamble_size; + const auto new_block_file_size = log_bundle.log_data.size() - nbytes_to_trim; + + boost::iostreams::mapped_file_sink new_block_file; + create_mapped_file(new_block_file, new_block_filename.generic_string(), new_block_file_size); + fc::datastream ds(new_block_file.data(), new_block_file.size()); + + block_log_preamble preamble; + // version 4 or above have different log entry format; therefore version 1 to 3 can only be upgrade up to version 3 format. + preamble.version = log_bundle.log_data.version() < pruned_transaction_version ? genesis_state_or_chain_id_version : block_log::max_supported_version; + preamble.first_block_num = truncate_at_block; + preamble.chain_context = log_bundle.log_data.chain_id(); + preamble.write_to(ds); + + memcpy(new_block_file.data() + preamble_size, log_bundle.log_data.data() + first_kept_block_pos, new_block_file_size - preamble_size); fc::path new_index_filename = temp_dir / "blocks.index"; - detail::index_writer index(new_index_filename, num_blocks); - - uint64_t read_size = 0; - for(uint64_t to_write_remaining = to_write; to_write_remaining > 0; to_write_remaining -= read_size) { - read_size = to_write_remaining; - if (read_size > detail::reverse_iterator::_buf_len) { - read_size = detail::reverse_iterator::_buf_len; - } - - // read in the previous contiguous memory into the read buffer - const auto start_of_blk_buffer_pos = original_file_block_pos + to_write_remaining - read_size; - status = fseek(original_block_log.blk_in, start_of_blk_buffer_pos, SEEK_SET); - const auto num_read = fread(buf, read_size, 1, original_block_log.blk_in); - EOS_ASSERT( num_read == 1, block_log_exception, "blocks.log read failed" ); - - // walk this memory section to adjust block position to match the adjusted location - // of the block start and store in the new index file - while(original_pos >= start_of_blk_buffer_pos) { - const auto buffer_index = original_pos - start_of_blk_buffer_pos; - uint64_t& pos_content = *(uint64_t*)(buf + buffer_index); - const auto start_of_this_block = pos_content; - pos_content = start_of_this_block - pos_delta; - index.write(pos_content); - original_pos = start_of_this_block - pos_size; - } - new_block_file.seek(new_block_file_first_block_pos + to_write_remaining - read_size); - new_block_file.write(buf, read_size); + index_writer index(new_index_filename, log_bundle.log_index.num_blocks() - num_blocks_to_truncate); + + // walk along the block position of each block entry and decrement its value by nbytes_to_trim + for (auto itr = make_reverse_block_position_iterator(new_block_file, preamble_size); + itr.get_value() != block_log::npos; ++itr) { + auto new_pos = itr.get_value() - nbytes_to_trim; + index.write(new_pos); + itr.set_value(new_pos); } - index.complete(); - fclose(original_block_log.blk_in); - original_block_log.blk_in = nullptr; - new_block_file.flush(); + + index.close(); new_block_file.close(); fc::path old_log = temp_dir / "old.log"; - rename(original_block_log.block_file_name, old_log); - rename(new_block_filename, original_block_log.block_file_name); + rename(log_bundle.block_file_name, old_log); + rename(new_block_filename, log_bundle.block_file_name); fc::path old_ind = temp_dir / "old.index"; - rename(original_block_log.index_file_name, old_ind); - rename(new_index_filename, original_block_log.index_file_name); + rename(log_bundle.index_file_name, old_ind); + rename(new_index_filename, log_bundle.index_file_name); return true; } - trim_data::trim_data(fc::path block_dir) { - - // code should follow logic in block_log::repair_log - - using namespace std; - block_file_name = block_dir / "blocks.log"; - index_file_name = block_dir / "blocks.index"; - blk_in = FC_FOPEN(block_file_name.generic_string().c_str(), "rb"); - EOS_ASSERT( blk_in != nullptr, block_log_not_found, "cannot read file ${file}", ("file",block_file_name.string()) ); - ind_in = FC_FOPEN(index_file_name.generic_string().c_str(), "rb"); - EOS_ASSERT( ind_in != nullptr, block_log_not_found, "cannot read file ${file}", ("file",index_file_name.string()) ); - auto size = fread((void*)&version,sizeof(version), 1, blk_in); - EOS_ASSERT( size == 1, block_log_unsupported_version, "invalid format for file ${file}", ("file",block_file_name.string())); - ilog("block log version= ${version}",("version",version)); - EOS_ASSERT( block_log::is_supported_version(version), block_log_unsupported_version, "block log version ${v} is not supported", ("v",version)); - - detail::fileptr_datastream ds(blk_in, block_file_name.string()); - if (version == 1) { - first_block = 1; - genesis_state gs; - fc::raw::unpack(ds, gs); - chain_id = gs.compute_chain_id(); - } - else { - size = fread((void *) &first_block, sizeof(first_block), 1, blk_in); - EOS_ASSERT(size == 1, block_log_exception, "invalid format for file ${file}", - ("file", block_file_name.string())); - if (block_log::contains_genesis_state(version, first_block)) { - genesis_state gs; - fc::raw::unpack(ds, gs); - chain_id = gs.compute_chain_id(); - } - else if (block_log::contains_chain_id(version, first_block)) { - ds >> chain_id; - } - else { - EOS_THROW( block_log_exception, - "Block log ${file} is not supported. version: ${ver} and first_block: ${first_block} does not contain " - "a genesis_state nor a chain_id.", - ("file", block_file_name.string())("ver", version)("first_block", first_block)); - } + int block_log::trim_blocklog_end(fc::path block_dir, uint32_t n) { //n is last block to keep (remove later blocks) + + block_log_bundle log_bundle(block_dir); - const auto expected_totem = block_log::npos; - std::decay_t actual_totem; - size = fread ( (char*)&actual_totem, sizeof(actual_totem), 1, blk_in); + ilog("In directory ${block_dir} will trim all blocks after block ${n} from ${block_file} and ${index_file}", + ("block_dir", block_dir.generic_string())("n", n)("block_file",log_bundle.block_file_name.generic_string())("index_file", log_bundle.index_file_name.generic_string())); - EOS_ASSERT(size == 1, block_log_exception, - "Expected to read ${size} bytes, but did not read any bytes", ("size", sizeof(actual_totem))); - EOS_ASSERT(actual_totem == expected_totem, block_log_exception, - "Expected separator between block log header and blocks was not found( expected: ${e}, actual: ${a} )", - ("e", fc::to_hex((char*)&expected_totem, sizeof(expected_totem) ))("a", fc::to_hex((char*)&actual_totem, sizeof(actual_totem) ))); + if (n < log_bundle.log_data.first_block_num()) { + dlog("All blocks are after block ${n} so do nothing (trim_end would delete entire blocks.log)",("n", n)); + return 1; + } + if (n > log_bundle.log_data.last_block_num()) { + dlog("There are no blocks after block ${n} so do nothing",("n", n)); + return 2; } - const uint64_t start_of_blocks = ftell(blk_in); - - const auto status = fseek(ind_in, 0, SEEK_END); //get length of blocks.index (gives number of blocks) - EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} end", ("file", index_file_name.string()) ); - const uint64_t file_end = ftell(ind_in); //get length of blocks.index (gives number of blocks) - last_block = first_block + file_end/sizeof(uint64_t) - 1; + const auto to_trim_block_index = n + 1 - log_bundle.log_data.first_block_num(); + const auto to_trim_block_position = log_bundle.log_index.nth_block_position(to_trim_block_index); + const auto index_file_size = to_trim_block_index * sizeof(uint64_t); - first_block_pos = block_pos(first_block); - EOS_ASSERT(start_of_blocks == first_block_pos, block_log_exception, - "Block log ${file} was determined to have its first block at ${determined}, but the block index " - "indicates the first block is at ${index}", - ("file", block_file_name.string())("determined", start_of_blocks)("index",first_block_pos)); - ilog("first block= ${first}",("first",first_block)); - ilog("last block= ${last}",("last",last_block)); + boost::filesystem::resize_file(log_bundle.block_file_name, to_trim_block_position); + boost::filesystem::resize_file(log_bundle.index_file_name, index_file_size); + ilog("blocks.index has been trimmed to ${index_file_size} bytes", ("index_file_size", index_file_size)); + return 0; } - trim_data::~trim_data() { - if (blk_in != nullptr) - fclose(blk_in); - if (ind_in != nullptr) - fclose(ind_in); - } + void block_log::smoke_test(fc::path block_dir, uint32_t interval) { - uint64_t trim_data::block_index(uint32_t n) const { - using namespace std; - EOS_ASSERT( first_block <= n, block_log_exception, - "cannot seek in ${file} to block number ${b}, block number ${first} is the first block", - ("file", index_file_name.string())("b",n)("first",first_block) ); - EOS_ASSERT( n <= last_block, block_log_exception, - "cannot seek in ${file} to block number ${b}, block number ${last} is the last block", - ("file", index_file_name.string())("b",n)("last",last_block) ); - return sizeof(uint64_t) * (n - first_block); - } + block_log_bundle log_bundle(block_dir); + + ilog("blocks.log and blocks.index agree on number of blocks"); - uint64_t trim_data::block_pos(uint32_t n) { - using namespace std; - // can indicate the location of the block after the last block - if (n == last_block + 1) { - return ftell(blk_in); + if (interval == 0) { + interval = std::max((log_bundle.log_index.num_blocks() + 7) >> 3, 1); + } + uint32_t expected_block_num = log_bundle.log_data.first_block_num(); + + for (auto pos_itr = log_bundle.log_index.begin(); pos_itr < log_bundle.log_index.end(); + pos_itr += interval, expected_block_num += interval) { + log_bundle.log_data.light_validate_block_entry_at(*pos_itr, expected_block_num); } - const uint64_t index_pos = block_index(n); - auto status = fseek(ind_in, index_pos, SEEK_SET); - EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file for block ${b}", ("file", index_file_name.string())("pos", index_pos)("b",n) ); - const uint64_t pos = ftell(ind_in); - EOS_ASSERT( pos == index_pos, block_log_exception, "cannot seek to ${file} entry for block ${b}", ("file", index_file_name.string())("b",n) ); - uint64_t block_n_pos; - auto size = fread((void*)&block_n_pos, sizeof(block_n_pos), 1, ind_in); //filepos of block n - EOS_ASSERT( size == 1, block_log_exception, "cannot read ${file} entry for block ${b}", ("file", index_file_name.string())("b",n) ); - - //read blocks.log and verify block number n is found at the determined file position - const auto calc_blknum_pos = block_n_pos + blknum_offset; - status = fseek(blk_in, calc_blknum_pos, SEEK_SET); - EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", block_file_name.string())("pos", calc_blknum_pos) ); - const uint64_t block_offset_pos = ftell(blk_in); - EOS_ASSERT( block_offset_pos == calc_blknum_pos, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", block_file_name.string())("pos", calc_blknum_pos) ); - uint32_t prior_blknum; - size = fread((void*)&prior_blknum, sizeof(prior_blknum), 1, blk_in); //read bigendian block number of prior block - EOS_ASSERT( size == 1, block_log_exception, "cannot read prior block"); - const uint32_t bnum = fc::endian_reverse_u32(prior_blknum) + 1; //convert to little endian, add 1 since prior block - EOS_ASSERT( bnum == n, block_log_exception, - "At position ${pos} in ${file} expected to find ${exp_bnum} but found ${act_bnum}", - ("pos",block_offset_pos)("file", block_file_name.string())("exp_bnum",n)("act_bnum",bnum) ); - - return block_n_pos; } - } } /// eosio::chain + bool block_log::exists(const fc::path& data_dir) { + return fc::exists(data_dir / "blocks.log") && fc::exists(data_dir / "blocks.index"); + } +}} /// eosio::chain diff --git a/libraries/chain/block_state.cpp b/libraries/chain/block_state.cpp index 8a616ff305b..fd614a61181 100644 --- a/libraries/chain/block_state.cpp +++ b/libraries/chain/block_state.cpp @@ -22,7 +22,7 @@ namespace eosio { namespace chain { auto exts = b->validate_and_extract_extensions(); if ( exts.count(additional_sigs_eid) > 0 ) { - auto& additional_sigs = exts.lower_bound(additional_sigs_eid)->second.get(); + auto& additional_sigs = std::get(exts.lower_bound(additional_sigs_eid)->second); return std::move(additional_sigs.signatures); } @@ -88,7 +88,7 @@ namespace eosio { namespace chain { block_state::block_state( pending_block_header_state&& cur, signed_block_ptr&& b, - vector&& trx_metas, + deque&& trx_metas, const protocol_feature_set& pfs, const std::function&, diff --git a/libraries/chain/chain_config.cpp b/libraries/chain/chain_config.cpp index 5aa72c2b9b1..db7d52d3b2e 100644 --- a/libraries/chain/chain_config.cpp +++ b/libraries/chain/chain_config.cpp @@ -1,9 +1,10 @@ #include #include +#include namespace eosio { namespace chain { - void chain_config::validate()const { + void chain_config_v0::validate() const { EOS_ASSERT( target_block_net_usage_pct <= config::percent_100, action_validate_exception, "target block net usage percentage cannot exceed 100%" ); EOS_ASSERT( target_block_net_usage_pct >= config::percent_1/10, action_validate_exception, @@ -38,4 +39,26 @@ namespace eosio { namespace chain { "max authority depth should be at least 1" ); } +void chain_config_v1::validate() const { + chain_config_v0::validate(); + EOS_ASSERT( max_action_return_value_size <= MAX_SIZE_OF_BYTE_ARRAYS, action_validate_exception, + "max action return value size should be less than MAX_SIZE_OF_BYTE_ARRAYS" ); +} + +bool config_entry_validator::operator()(uint32_t id) const { + bool allowed = true; + switch(id){ + case chain_config_v1::max_action_return_value_size_id: + { + allowed = control.is_builtin_activated(builtin_protocol_feature_t::action_return_value); + if (!allowed){ + wlog("action_return_value protocol feature is not active, max_action_return_value_size config is not allowed"); + } + } + break; + } + + return allowed; +} + } } // namespace eosio::chain diff --git a/libraries/chain/combined_database.cpp b/libraries/chain/combined_database.cpp new file mode 100644 index 00000000000..cf7a674a879 --- /dev/null +++ b/libraries/chain/combined_database.cpp @@ -0,0 +1,688 @@ +#include +#include +#include +#include +#include +#include +#include + +namespace eosio { namespace chain { + combined_session::combined_session(chainbase::database& cb_database, eosio::session::undo_stack* undo_stack) + : kv_undo_stack{ undo_stack } { + cb_session = std::make_unique(cb_database.start_undo_session(true)); + try { + try { + if (kv_undo_stack) { + kv_undo_stack->push(); + } + } + FC_LOG_AND_RETHROW() + } + CATCH_AND_EXIT_DB_FAILURE() + } + + combined_session::combined_session(combined_session&& src) noexcept + : cb_session(std::move(src.cb_session)), kv_undo_stack(src.kv_undo_stack) { + src.kv_undo_stack = nullptr; + } + + void combined_session::push() { + if (cb_session) { + cb_session->push(); + cb_session = nullptr; + + if (kv_undo_stack) { + kv_undo_stack = nullptr; + } + } + } + + void combined_session::squash() { + if (cb_session) { + cb_session->squash(); + cb_session = nullptr; + + if (kv_undo_stack) { + try { + try { + kv_undo_stack->squash(); + kv_undo_stack = nullptr; + } + FC_LOG_AND_RETHROW() + } + CATCH_AND_EXIT_DB_FAILURE() + } + } + } + + void combined_session::undo() { + if (cb_session) { + cb_session->undo(); + cb_session = nullptr; + + if (kv_undo_stack) { + try { + try { + kv_undo_stack->undo(); + kv_undo_stack = nullptr; + } + FC_LOG_AND_RETHROW() + } + CATCH_AND_EXIT_DB_FAILURE() + } + } + } + + template + void walk_index(const Util& utils, const chainbase::database& db, F&& function) { + utils.walk(db, std::forward(function)); + } + + template + void walk_index(const index_utils& utils, const chainbase::database& db, F&& function) {} + + template + void walk_index(const index_utils& utils, const chainbase::database& db, F&& function) {} + + template + void walk_index(const index_utils& utils, const chainbase::database& db, F&& function) {} + + void add_kv_table_to_snapshot(const snapshot_writer_ptr& snapshot, const chainbase::database& db, const kv_undo_stack_ptr& kv_undo_stack) { + snapshot->write_section([&db,&kv_undo_stack](auto& section) { + if (kv_undo_stack && db.get().backing_store == backing_store_type::ROCKSDB) { + using add_database_section_receiver = backing_store::add_database_receiver>; + add_database_section_receiver add_db_receiver(section, db); + backing_store::rocksdb_contract_kv_table_writer writer(add_db_receiver); + const auto begin_key = eosio::session::shared_bytes(&backing_store::rocksdb_contract_kv_prefix, 1); + const auto end_key = begin_key.next(); + backing_store::walk_rocksdb_entries_with_prefix(kv_undo_stack, begin_key, end_key, writer); + } else { + index_utils utils; + utils.walk(db, [&db, §ion](const auto& row) { section.add_row(row, db); }); + } + }); + } + + void read_kv_table_from_snapshot(const snapshot_reader_ptr& snapshot, chainbase::database& db, + const std::unique_ptr& kv_database, uint32_t version, backing_store_type backing_store ) { + if (version < kv_object::minimum_snapshot_version) + return; + if (backing_store == backing_store_type::ROCKSDB) { + auto key_values = std::vector>{}; + constexpr std::size_t batch_size = 500; + key_values.reserve(batch_size); + snapshot->read_section([&key_values, &db, &kv_database](auto& section) { + const std::string_view prefix_key {&backing_store::rocksdb_contract_kv_prefix, 1}; + bool more = !section.empty(); + while (more) { + kv_object_view move_to_rocks; + more = section.read_row(move_to_rocks, db); + b1::chain_kv::bytes contract_as_bytes; + b1::chain_kv::append_key(contract_as_bytes, move_to_rocks.contract.to_uint64_t()); + auto full_key = + eosio::session::make_shared_bytes({prefix_key, + std::string_view{contract_as_bytes.data(), + contract_as_bytes.size()}, + std::string_view{move_to_rocks.kv_key.data.data(), + move_to_rocks.kv_key.data.size()}}); + + // Pack payer and actual key value + auto final_kv_value = backing_store::payer_payload(move_to_rocks.payer, + move_to_rocks.kv_value.data.data(), + move_to_rocks.kv_value.data.size()); + + key_values.emplace_back(full_key, + final_kv_value.as_payload()); + + if (key_values.size() >= batch_size) { + kv_database->write(key_values); + key_values.clear(); + } + } + }); + // write out any remaining key-values + kv_database->write(key_values); + } + else { + snapshot->read_section([&db](auto& section) { + bool more = !section.empty(); + while (more) { + index_utils::create(db, [&db, §ion, &more](auto &row) { more = section.read_row(row, db); }); + } + }); + } + } + + combined_database::combined_database(chainbase::database& chain_db, + uint32_t snapshot_batch_threashold) + : backing_store(backing_store_type::CHAINBASE), db(chain_db), kv_snapshot_batch_threashold(snapshot_batch_threashold * 1024 * 1024) {} + + combined_database::combined_database(chainbase::database& chain_db, + const controller::config& cfg) + : backing_store(backing_store_type::ROCKSDB), db(chain_db), kv_database{ [&]() { + rocksdb::Options options; + + options.create_if_missing = true; // Creates a database if it is missing + options.level_compaction_dynamic_level_bytes = true; + options.bytes_per_sync = cfg.persistent_storage_bytes_per_sync; // used to control the write rate of flushes and compactions. + options.use_adaptive_mutex = true; + + // Number of threads used for flush and compaction. + options.IncreaseParallelism(cfg.persistent_storage_num_threads); + + options.OptimizeLevelStyleCompaction(512ull << 20); // optimizes level style compaction + + // Number of open files that can be used by the DB. + // Setting it to -1 means files opened are always kept open. + options.max_open_files = cfg.persistent_storage_max_num_files; + + // Use this option to increase the number of threads + // used to open the files. + options.max_file_opening_threads = cfg.persistent_storage_num_threads; + + // Write Buffer Size - Sets the size of a single + // memtable. Once memtable exceeds this size, it is + // marked immutable and a new one is created. + // Default should be 128MB + options.write_buffer_size = cfg.persistent_storage_write_buffer_size; + options.max_write_buffer_number = 10; // maximum number of memtables, both active and immutable + options.min_write_buffer_number_to_merge = 2; // minimum number of memtables to be merged before flushing to storage + + // Once level 0 reaches this number of files, L0->L1 compaction is triggered. + options.level0_file_num_compaction_trigger = 2; + + // Size of L0 = write_buffer_size * min_write_buffer_number_to_merge * level0_file_num_compaction_trigger + // For optimal performance make this equal to L0 + options.max_bytes_for_level_base = cfg.persistent_storage_write_buffer_size * options.min_write_buffer_number_to_merge * options.level0_file_num_compaction_trigger; + + // Files in level 1 will have target_file_size_base + // bytes. It’s recommended setting target_file_size_base + // to be max_bytes_for_level_base / 10. + options.target_file_size_base = options.max_bytes_for_level_base / 10; + + // This value represents the maximum number of threads + // that will concurrently perform a compaction job by + // breaking it into multiple, + // smaller ones that are run simultaneously. + options.max_subcompactions = cfg.persistent_storage_num_threads; + + // Full and partitioned filters in the block-based table + // use an improved Bloom filter implementation, enabled + // with format_version 5 (or above) because previous + // releases cannot read this filter. This replacement is + // faster and more accurate, especially for high bits + // per key or millions of keys in a single (full) filter. + rocksdb::BlockBasedTableOptions table_options; + table_options.format_version = 5; + table_options.index_block_restart_interval = 16; + + // Sets the bloom filter - Given an arbitrary key, + // this bit array may be used to determine if the key + // may exist or definitely does not exist in the key set. + table_options.filter_policy.reset(rocksdb::NewBloomFilterPolicy(15, false)); + table_options.index_type = rocksdb::BlockBasedTableOptions::kBinarySearch; + + // Incorporates the Table options into options + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + rocksdb::DB* p; + auto status = rocksdb::DB::Open(options, (cfg.state_dir / "chain-kv").string(), &p); + if (!status.ok()) + throw std::runtime_error(std::string{ "database::database: rocksdb::DB::Open: " } + status.ToString()); + auto rdb = std::shared_ptr{ p }; + return std::make_unique(eosio::session::make_session(std::move(rdb), 1024)); + }() }, + kv_undo_stack(std::make_unique>(*kv_database, cfg.state_dir)), + kv_snapshot_batch_threashold(cfg.persistent_storage_mbytes_batch * 1024 * 1024) {} + + void combined_database::check_backing_store_setting(bool clean_startup) { + if (backing_store != db.get().backing_store) { + EOS_ASSERT(clean_startup, database_move_kv_disk_exception, + "Existing state indicates a different backing store is in use; use resync, replay, or restore from snapshot to switch backing store"); + db.modify(db.get(), [this](auto& cfg) { cfg.backing_store = backing_store; }); + } + + if (backing_store == backing_store_type::ROCKSDB) + ilog("using rocksdb for backing store"); + else + ilog("using chainbase for backing store"); + } + + void combined_database::destroy(const fc::path& p) { + if( !fc::is_directory( p ) ) + return; + + fc::remove( p / "shared_memory.bin" ); + fc::remove( p / "shared_memory.meta" ); + + rocks_db_type::destroy((p / "chain-kv").string()); + } + + void combined_database::set_revision(uint64_t revision) { + db.set_revision(revision); + + if (backing_store == backing_store_type::ROCKSDB) { + try { + try { + kv_undo_stack->revision(revision); + } + FC_LOG_AND_RETHROW() + } + CATCH_AND_EXIT_DB_FAILURE() + } + } + + int64_t combined_database::revision() { + if (backing_store == backing_store_type::ROCKSDB) { + try { + try { + return kv_undo_stack->revision(); + } + FC_LOG_AND_RETHROW() + } + CATCH_AND_EXIT_DB_FAILURE() + } else { + return db.revision(); + } + } + + void combined_database::undo() { + db.undo(); + + if (backing_store == backing_store_type::ROCKSDB) { + try { + try { + kv_undo_stack->undo(); + } + FC_LOG_AND_RETHROW() + } + CATCH_AND_EXIT_DB_FAILURE() + } + } + + void combined_database::commit(int64_t revision) { + db.commit(revision); + + if (backing_store == backing_store_type::ROCKSDB) { + try { + try { + kv_undo_stack->commit(revision); + } + FC_LOG_AND_RETHROW() + } + CATCH_AND_EXIT_DB_FAILURE() + } + } + + void combined_database::flush() { + if (backing_store == backing_store_type::ROCKSDB) { + try { + try { + kv_database->flush(); + } + FC_LOG_AND_RETHROW() + } + CATCH_AND_EXIT_DB_FAILURE() + } + } + + std::unique_ptr combined_database::create_kv_context(name receiver, kv_resource_manager resource_manager, + const kv_database_config& limits) const { + switch (backing_store) { + case backing_store_type::ROCKSDB: + return std::visit([&](auto* session){ + return create_kv_rocksdb_context, kv_resource_manager>(*session, receiver, + resource_manager, limits); + }, kv_undo_stack->top().holder()); + case backing_store_type::CHAINBASE: + return create_kv_chainbase_context(db, receiver, resource_manager, limits); + } + EOS_ASSERT(false, action_validate_exception, "Unknown backing store."); + } + + std::unique_ptr combined_database::create_db_context(apply_context& context, name receiver) { + switch (backing_store) { + case backing_store_type::ROCKSDB: + return backing_store::create_db_rocksdb_context(context, receiver, kv_undo_stack->top()); + case backing_store_type::CHAINBASE: + return backing_store::create_db_chainbase_context(context, receiver); + default: + EOS_ASSERT(false, action_validate_exception, "Unknown backing store."); + } + } + + void combined_database::add_to_snapshot( + const eosio::chain::snapshot_writer_ptr& snapshot, const eosio::chain::block_state& head, + const eosio::chain::authorization_manager& authorization, + const eosio::chain::resource_limits::resource_limits_manager& resource_limits) const { + snapshot->write_section( + [this](auto& section) { section.add_row(chain_snapshot_header(), db); }); + + snapshot->write_section( + [this, &head](auto& section) { section.template add_row(head, db); }); + + eosio::chain::controller_index_set::walk_indices([this, &snapshot](auto utils) { + using value_t = typename decltype(utils)::index_t::value_type; + + snapshot->write_section([utils, this](auto& section) { + walk_index(utils, db, [this, §ion](const auto& row) { section.add_row(row, db); }); + }); + }); + + add_kv_table_to_snapshot(snapshot, db, kv_undo_stack); + add_contract_tables_to_snapshot(snapshot); + + authorization.add_to_snapshot(snapshot); + resource_limits.add_to_snapshot(snapshot); + } + + void combined_database::read_from_snapshot(const snapshot_reader_ptr& snapshot, + uint32_t blog_start, + uint32_t blog_end, + eosio::chain::authorization_manager& authorization, + eosio::chain::resource_limits::resource_limits_manager& resource_limits, + eosio::chain::fork_database& fork_db, eosio::chain::block_state_ptr& head, + uint32_t& snapshot_head_block, + const eosio::chain::chain_id_type& chain_id) { + chain_snapshot_header header; + snapshot->read_section([this, &header](auto& section) { + section.read_row(header, db); + header.validate(); + }); + + db.create([](auto&) {}); + check_backing_store_setting(true); + + { /// load and upgrade the block header state + block_header_state head_header_state; + using v2 = legacy::snapshot_block_header_state_v2; + + if (std::clamp(header.version, v2::minimum_version, v2::maximum_version) == header.version) { + snapshot->read_section([this, &head_header_state](auto& section) { + legacy::snapshot_block_header_state_v2 legacy_header_state; + section.read_row(legacy_header_state, db); + head_header_state = block_header_state(std::move(legacy_header_state)); + }); + } else { + snapshot->read_section( + [this, &head_header_state](auto& section) { section.read_row(head_header_state, db); }); + } + + snapshot_head_block = head_header_state.block_num; + EOS_ASSERT(blog_start <= (snapshot_head_block + 1) && snapshot_head_block <= blog_end, block_log_exception, + "Block log is provided with snapshot but does not contain the head block from the snapshot nor a " + "block right after it", + ("snapshot_head_block", snapshot_head_block)("block_log_first_num", + blog_start)("block_log_last_num", blog_end)); + + fork_db.reset(head_header_state); + head = fork_db.head(); + snapshot_head_block = head->block_num; + } + + controller_index_set::walk_indices([this, &snapshot, &header](auto utils) { + using value_t = typename decltype(utils)::index_t::value_type; + + // skip the table_id_object as its inlined with contract tables section + if (std::is_same::value) { + return; + } + + // skip the database_header as it is only relevant to in-memory database + if (std::is_same::value) { + return; + } + + // skip the kv_db_config as it only determines where the kv-database is stored + if (std::is_same_v) { + return; + } + + // special case for in-place upgrade of global_property_object + if (std::is_same::value) { + using v2 = legacy::snapshot_global_property_object_v2; + using v3 = legacy::snapshot_global_property_object_v3; + using v4 = legacy::snapshot_global_property_object_v4; + + if (std::clamp(header.version, v2::minimum_version, v2::maximum_version) == header.version) { + std::optional genesis = extract_legacy_genesis_state(*snapshot, header.version); + EOS_ASSERT(genesis, snapshot_exception, + "Snapshot indicates chain_snapshot_header version 2, but does not contain a genesis_state. " + "It must be corrupted."); + snapshot->read_section( + [&db = this->db, gs_chain_id = genesis->compute_chain_id()](auto& section) { + v2 legacy_global_properties; + section.read_row(legacy_global_properties, db); + + db.create([&legacy_global_properties, &gs_chain_id](auto& gpo) { + gpo.initalize_from(legacy_global_properties, gs_chain_id, kv_database_config{}, + genesis_state::default_initial_wasm_configuration); + }); + }); + return; // early out to avoid default processing + } + + if (std::clamp(header.version, v3::minimum_version, v3::maximum_version) == header.version) { + snapshot->read_section([&db = this->db](auto& section) { + v3 legacy_global_properties; + section.read_row(legacy_global_properties, db); + + db.create([&legacy_global_properties](auto& gpo) { + gpo.initalize_from(legacy_global_properties, kv_database_config{}, + genesis_state::default_initial_wasm_configuration); + }); + }); + return; // early out to avoid default processing + } + + if (std::clamp(header.version, v4::minimum_version, v4::maximum_version) == header.version) { + snapshot->read_section([&db = this->db](auto& section) { + v4 legacy_global_properties; + section.read_row(legacy_global_properties, db); + + db.create([&legacy_global_properties](auto& gpo) { + gpo.initalize_from(legacy_global_properties); + }); + }); + return; // early out to avoid default processing + } + + } + + snapshot->read_section([this](auto& section) { + bool more = !section.empty(); + while (more) { + decltype(utils)::create(db, [this, §ion, &more](auto& row) { more = section.read_row(row, db); }); + } + }); + }); + + read_kv_table_from_snapshot(snapshot, db, kv_database, header.version, backing_store); + read_contract_tables_from_snapshot(snapshot); + + authorization.read_from_snapshot(snapshot); + resource_limits.read_from_snapshot(snapshot, header.version); + + set_revision(head->block_num); + db.create([](const auto& header) { + // nothing to do + }); + + const auto& gpo = db.get(); + EOS_ASSERT(gpo.chain_id == chain_id, chain_id_type_exception, + "chain ID in snapshot (${snapshot_chain_id}) does not match the chain ID that controller was " + "constructed with (${controller_chain_id})", + ("snapshot_chain_id", gpo.chain_id)("controller_chain_id", chain_id)); + } + + template + void chainbase_add_contract_tables_to_snapshot(const chainbase::database& db, Section& section) { + index_utils::walk(db, [&db, §ion](const table_id_object& table_row) { + // add a row for the table + section.add_row(table_row, db); + + // followed by a size row and then N data rows for each type of table + contract_database_index_set::walk_indices([&db, §ion, &table_row](auto utils) { + using utils_t = decltype(utils); + using value_t = typename utils_t::index_t::value_type; + using by_table_id = object_to_table_id_tag_t; + + auto tid_key = std::make_tuple(table_row.id); + auto next_tid_key = std::make_tuple(table_id_object::id_type(table_row.id._id + 1)); + + unsigned_int size = utils_t::template size_range(db, tid_key, next_tid_key); + section.add_row(size, db); + + utils_t::template walk_range(db, tid_key, next_tid_key, [&db, §ion](const auto& row) { + section.add_row(row, db); + }); + }); + }); + } + + void combined_database::add_contract_tables_to_snapshot(const snapshot_writer_ptr& snapshot) const { + snapshot->write_section("contract_tables", [this](auto& section) { + if (kv_undo_stack && db.get().backing_store == backing_store_type::ROCKSDB) { + using add_database_section_receiver = backing_store::add_database_receiver>; + using table_collector = backing_store::rocksdb_whole_db_table_collector; + + add_database_section_receiver add_db_receiver(section, db); + table_collector table_collector_receiver(add_db_receiver); + backing_store::rocksdb_contract_db_table_writer writer(table_collector_receiver); + const auto begin_key = eosio::session::shared_bytes(&backing_store::rocksdb_contract_db_prefix, 1); + const auto end_key = begin_key.next(); + backing_store::walk_rocksdb_entries_with_prefix(kv_undo_stack, begin_key, end_key, writer); + } + else { + chainbase_add_contract_tables_to_snapshot(db, section); + } + }); + } + + template + void chainbase_read_contract_tables_from_snapshot(chainbase::database& db, Section& section) { + bool more = !section.empty(); + while (more) { + // read the row for the table + table_id_object::id_type t_id; + + index_utils::create(db, [&db, §ion, &t_id](auto& row) { + section.read_row(row, db); + t_id = row.id; + }); + + // read the size and data rows for each type of table + contract_database_index_set::walk_indices([&db, §ion, &t_id, &more](auto utils) { + using utils_t = decltype(utils); + + unsigned_int size; + more = section.read_row(size, db); + + for (size_t idx = 0; idx < size.value; ++idx) { + utils_t::create(db, [&db, §ion, &more, &t_id](auto& row) { + row.t_id = t_id; + more = section.read_row(row, db); + }); + } + }); + } + } + + template + void rocksdb_read_contract_tables_from_snapshot(rocks_db_type& kv_database, chainbase::database& db, + Section& section, uint64_t snapshot_batch_threashold) { + std::vector> batch; + bool more = !section.empty(); + auto read_row = [§ion, &more, &db](auto& row) { more = section.read_row(row, db); }; + uint64_t batch_mem_size = 0; + + while (more) { + // read the row for the table + backing_store::table_id_object_view table_obj; + read_row(table_obj); + auto put = [&batch, &table_obj, &batch_mem_size, &kv_database, snapshot_batch_threashold] + (auto&& value, auto create_fun, auto&&... args) { + auto composite_key = create_fun(table_obj.scope, table_obj.table, std::forward(args)...); + batch.emplace_back(backing_store::db_key_value_format::create_full_key(composite_key, table_obj.code), + std::forward(value)); + + const auto& back = batch.back(); + const auto size = back.first.size() + back.second.size(); + if (size >= snapshot_batch_threashold || snapshot_batch_threashold - size < batch_mem_size) { + kv_database.write(batch); + batch_mem_size = 0; + batch.clear(); + } + else { + batch_mem_size += size; + } + }; + + // handle the primary key index + unsigned_int size; + read_row(size); + for (size_t i = 0; i < size.value; ++i) { + backing_store::primary_index_view row; + read_row(row); + backing_store::payer_payload pp{row.payer, row.value.data(), row.value.size()}; + put(pp.as_payload(), backing_store::db_key_value_format::create_primary_key, row.primary_key); + } + + auto write_secondary_index = [&put, &read_row](auto index) { + using index_t = decltype(index); + static const eosio::session::shared_bytes empty_payload; + unsigned_int size; + read_row(size); + for (uint32_t i = 0; i < size.value; ++i) { + backing_store::secondary_index_view row; + read_row(row); + backing_store::payer_payload pp{row.payer, nullptr, 0}; + put(pp.as_payload(), &backing_store::db_key_value_format::create_secondary_key, + row.secondary_key, row.primary_key); + + put(empty_payload, &backing_store::db_key_value_format::create_primary_to_secondary_key, + row.primary_key, row.secondary_key); + } + }; + + // handle secondary key indices + std::tuple indices; + std::apply([&write_secondary_index](auto... index) { (write_secondary_index(index), ...); }, indices); + + backing_store::payer_payload pp{table_obj.payer, nullptr, 0}; + b1::chain_kv::bytes (*create_table_key)(name scope, name table) = backing_store::db_key_value_format::create_table_key; + put(pp.as_payload(), create_table_key); + + } + kv_database.write(batch); + } + + void combined_database::read_contract_tables_from_snapshot(const snapshot_reader_ptr& snapshot) { + snapshot->read_section("contract_tables", [this](auto& section) { + if (kv_undo_stack && db.get().backing_store == backing_store_type::ROCKSDB) + rocksdb_read_contract_tables_from_snapshot(*kv_database, db, section, kv_snapshot_batch_threashold); + else + chainbase_read_contract_tables_from_snapshot(db, section); + }); + } + + std::optional extract_legacy_genesis_state(snapshot_reader& snapshot, + uint32_t version) { + std::optional genesis; + using v2 = legacy::snapshot_global_property_object_v2; + + if (std::clamp(version, v2::minimum_version, v2::maximum_version) == version) { + genesis.emplace(); + snapshot.read_section( + [&genesis = *genesis](auto& section) { section.read_row(genesis); }); + } + return genesis; + } + + // TODO : need to change this method to just return a char + std::vector make_rocksdb_contract_kv_prefix() { return std::vector { backing_store::rocksdb_contract_kv_prefix }; } + char make_rocksdb_contract_db_prefix() { return backing_store::rocksdb_contract_db_prefix; } + +}} // namespace eosio::chain + + diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 807260e6d86..587eb8b28e0 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -5,24 +5,11 @@ #include #include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include #include #include -#include #include #include @@ -31,82 +18,18 @@ #include #include #include +#include #include +#if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED) +#include +#endif + namespace eosio { namespace chain { using resource_limits::resource_limits_manager; -using controller_index_set = index_set< - account_index, - account_metadata_index, - account_ram_correction_index, - global_property_multi_index, - protocol_state_multi_index, - dynamic_global_property_multi_index, - block_summary_multi_index, - transaction_multi_index, - generated_transaction_multi_index, - table_id_multi_index, - code_index, - database_header_multi_index ->; - -using contract_database_index_set = index_set< - key_value_index, - index64_index, - index128_index, - index256_index, - index_double_index, - index_long_double_index ->; - -class maybe_session { - public: - maybe_session() = default; - - maybe_session( maybe_session&& other) - :_session(move(other._session)) - { - } - - explicit maybe_session(database& db) { - _session = db.start_undo_session(true); - } - - maybe_session(const maybe_session&) = delete; - - void squash() { - if (_session) - _session->squash(); - } - - void undo() { - if (_session) - _session->undo(); - } - - void push() { - if (_session) - _session->push(); - } - - maybe_session& operator = ( maybe_session&& mv ) { - if (mv._session) { - _session = move(*mv._session); - mv._session.reset(); - } else { - _session.reset(); - } - - return *this; - }; - - private: - optional _session; -}; - struct building_block { building_block( const block_header_state& prev, block_timestamp_type when, @@ -114,36 +37,37 @@ struct building_block { const vector& new_protocol_feature_activations ) :_pending_block_header_state( prev.next( when, num_prev_blocks_to_confirm ) ) ,_new_protocol_feature_activations( new_protocol_feature_activations ) + ,_trx_mroot_or_receipt_digests( digests_t{} ) {} - pending_block_header_state _pending_block_header_state; - optional _new_pending_producer_schedule; - vector _new_protocol_feature_activations; - size_t _num_new_protocol_features_that_have_activated = 0; - vector _pending_trx_metas; - vector _pending_trx_receipts; - vector _actions; - optional _transaction_mroot; + pending_block_header_state _pending_block_header_state; + std::optional _new_pending_producer_schedule; + vector _new_protocol_feature_activations; + size_t _num_new_protocol_features_that_have_activated = 0; + deque _pending_trx_metas; + deque _pending_trx_receipts; // boost deque in 1.71 with 1024 elements performs better + std::variant _trx_mroot_or_receipt_digests; + digests_t _action_receipt_digests; }; struct assembled_block { block_id_type _id; pending_block_header_state _pending_block_header_state; - vector _trx_metas; + deque _trx_metas; signed_block_ptr _unsigned_block; // if the _unsigned_block pre-dates block-signing authorities this may be present. - optional _new_producer_authority_cache; + std::optional _new_producer_authority_cache; }; struct completed_block { block_state_ptr _block_state; }; -using block_stage_type = fc::static_variant; +using block_stage_type = std::variant; struct pending_state { - pending_state( maybe_session&& s, const block_header_state& prev, + pending_state( combined_session&& s, const block_header_state& prev, block_timestamp_type when, uint16_t num_prev_blocks_to_confirm, const vector& new_protocol_feature_activations ) @@ -151,42 +75,42 @@ struct pending_state { ,_block_stage( building_block( prev, when, num_prev_blocks_to_confirm, new_protocol_feature_activations ) ) {} - maybe_session _db_session; + combined_session _db_session; block_stage_type _block_stage; controller::block_status _block_status = controller::block_status::incomplete; - optional _producer_block_id; + std::optional _producer_block_id; /** @pre _block_stage cannot hold completed_block alternative */ const pending_block_header_state& get_pending_block_header_state()const { - if( _block_stage.contains() ) - return _block_stage.get()._pending_block_header_state; + if( std::holds_alternative(_block_stage) ) + return std::get(_block_stage)._pending_block_header_state; - return _block_stage.get()._pending_block_header_state; + return std::get(_block_stage)._pending_block_header_state; } - const vector& get_trx_receipts()const { - if( _block_stage.contains() ) - return _block_stage.get()._pending_trx_receipts; + const deque& get_trx_receipts()const { + if( std::holds_alternative(_block_stage) ) + return std::get(_block_stage)._pending_trx_receipts; - if( _block_stage.contains() ) - return _block_stage.get()._unsigned_block->transactions; + if( std::holds_alternative(_block_stage) ) + return std::get(_block_stage)._unsigned_block->transactions; - return _block_stage.get()._block_state->block->transactions; - } + return std::get(_block_stage)._block_state->block->transactions; + } - vector extract_trx_metas() { - if( _block_stage.contains() ) - return std::move( _block_stage.get()._pending_trx_metas ); + deque extract_trx_metas() { + if( std::holds_alternative(_block_stage) ) + return std::move( std::get(_block_stage)._pending_trx_metas ); - if( _block_stage.contains() ) - return std::move( _block_stage.get()._trx_metas ); + if( std::holds_alternative(_block_stage) ) + return std::move( std::get(_block_stage)._trx_metas ); - return _block_stage.get()._block_state->extract_trxs_metas(); + return std::get(_block_stage)._block_state->extract_trxs_metas(); } bool is_protocol_feature_activated( const digest_type& feature_digest )const { - if( _block_stage.contains() ) { - auto& bb = _block_stage.get(); + if( std::holds_alternative(_block_stage) ) { + auto& bb = std::get(_block_stage); const auto& activated_features = bb._pending_block_header_state.prev_activated_protocol_features->protocol_features; if( activated_features.find( feature_digest ) != activated_features.end() ) return true; @@ -197,7 +121,7 @@ struct pending_state { return (std::find( bb._new_protocol_feature_activations.begin(), end, feature_digest ) != end); } - if( _block_stage.contains() ) { + if( std::holds_alternative(_block_stage) ) { // Calling is_protocol_feature_activated during the assembled_block stage is not efficient. // We should avoid doing it. // In fact for now it isn't even implemented. @@ -206,7 +130,7 @@ struct pending_state { // TODO: implement this } - const auto& activated_features = _block_stage.get()._block_state->activated_protocol_features->protocol_features; + const auto& activated_features = std::get(_block_stage)._block_state->activated_protocol_features->protocol_features; return (activated_features.find( feature_digest ) != activated_features.end()); } @@ -223,30 +147,33 @@ struct controller_impl { reset_new_handler() { std::set_new_handler([](){ throw std::bad_alloc(); }); } }; - reset_new_handler rnh; // placed here to allow for this to be set before constructing the other fields - controller& self; - chainbase::database db; - chainbase::database reversible_blocks; ///< a special database to persist blocks that have successfully been applied but are still reversible - block_log blog; - optional pending; - block_state_ptr head; - fork_database fork_db; - wasm_interface wasmif; - resource_limits_manager resource_limits; - authorization_manager authorization; - protocol_feature_manager protocol_features; - controller::config conf; - const chain_id_type chain_id; // read by thread_pool threads, value will not be changed - optional replay_head_time; - db_read_mode read_mode = db_read_mode::SPECULATIVE; - bool in_trx_requiring_checks = false; ///< if true, checks that are normally skipped on replay (e.g. auth checks) cannot be skipped - optional subjective_cpu_leeway; - bool trusted_producer_light_validation = false; - uint32_t snapshot_head_block = 0; - named_thread_pool thread_pool; - platform_timer timer; + reset_new_handler rnh; // placed here to allow for this to be set before constructing the other fields + controller& self; + std::function shutdown; + chainbase::database db; + chainbase::database reversible_blocks; ///< a special database to persist blocks that have successfully been applied but are still reversible + combined_database kv_db; + block_log blog; + std::optional pending; + block_state_ptr head; + fork_database fork_db; + wasm_interface wasmif; + resource_limits_manager resource_limits; + authorization_manager authorization; + protocol_feature_manager protocol_features; + controller::config conf; + const chain_id_type chain_id; // read by thread_pool threads, value will not be changed + std::optional replay_head_time; + db_read_mode read_mode = db_read_mode::SPECULATIVE; + bool in_trx_requiring_checks = false; ///< if true, checks that are normally skipped on replay (e.g. auth checks) cannot be skipped + std::optional subjective_cpu_leeway; + bool trusted_producer_light_validation = false; + uint32_t snapshot_head_block = 0; + named_thread_pool thread_pool; + platform_timer timer; + fc::logger* deep_mind_logger = nullptr; #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED) - vm::wasm_allocator wasm_alloc; + vm::wasm_allocator wasm_alloc; #endif typedef pair handler_key; @@ -272,7 +199,7 @@ struct controller_impl { head = prev; - db.undo(); + kv_db.undo(); protocol_features.popped_blocks_to( prev->block_num ); } @@ -301,16 +228,19 @@ struct controller_impl { self(s), db( cfg.state_dir, cfg.read_only ? database::read_only : database::read_write, - cfg.state_size, false, cfg.db_map_mode, cfg.db_hugepage_paths ), - reversible_blocks( cfg.blocks_dir/config::reversible_blocks_dir_name, + cfg.state_size, false, cfg.db_map_mode ), + reversible_blocks( cfg.blog.log_dir/config::reversible_blocks_dir_name, cfg.read_only ? database::read_only : database::read_write, - cfg.reversible_cache_size, false, cfg.db_map_mode, cfg.db_hugepage_paths ), - blog( cfg.blocks_dir ), + cfg.reversible_cache_size, false, cfg.db_map_mode ), + kv_db(cfg.backing_store == backing_store_type::CHAINBASE + ? combined_database(db, cfg.persistent_storage_mbytes_batch) + : combined_database(db, cfg)), + blog( cfg.blog ), fork_db( cfg.state_dir ), wasmif( cfg.wasm_runtime, cfg.eosvmoc_tierup, db, cfg.state_dir, cfg.eosvmoc_config ), - resource_limits( db ), + resource_limits( db, [&s]() { return s.get_deep_mind_logger(); }), authorization( s, db ), - protocol_features( std::move(pfs) ), + protocol_features( std::move(pfs), [&s]() { return s.get_deep_mind_logger(); } ), conf( cfg ), chain_id( chain_id ), read_mode( cfg.read_mode ), @@ -327,12 +257,15 @@ struct controller_impl { set_activation_handler(); set_activation_handler(); set_activation_handler(); + set_activation_handler(); + set_activation_handler(); + set_activation_handler(); + set_activation_handler(); self.irreversible_block.connect([this](const block_state_ptr& bsp) { wasmif.current_lib(bsp->block_num); }); - #define SET_APP_HANDLER( receiver, contract, action) \ set_apply_handler( account_name(#receiver), account_name(#contract), action_name(#action), \ &BOOST_PP_CAT(apply_, BOOST_PP_CAT(contract, BOOST_PP_CAT(_,action) ) ) ) @@ -394,10 +327,11 @@ struct controller_impl { auto root_id = fork_db.root()->id; if( log_head ) { - EOS_ASSERT( root_id == blog.head_id(), fork_database_exception, "fork database root does not match block log head" ); + // todo: move this check to startup so id does not have to be calculated + EOS_ASSERT( root_id == log_head->calculate_id(), fork_database_exception, "fork database root does not match block log head" ); } else { EOS_ASSERT( fork_db.root()->block_num == lib_num, fork_database_exception, - "empty block log expects the first appended block to build off a block that is not the fork database root" ); + "empty block log expects the first appended block to build off a block that is not the fork database root. root block number: ${block_num}, lib: ${lib_num}", ("block_num", fork_db.root()->block_num) ("lib_num", lib_num) ); } auto fork_head = (read_mode == db_read_mode::IRREVERSIBLE) ? fork_db.pending_head() : fork_db.head(); @@ -405,10 +339,18 @@ struct controller_impl { if( fork_head->dpos_irreversible_blocknum <= lib_num ) return; - const auto branch = fork_db.fetch_branch( fork_head->id, fork_head->dpos_irreversible_blocknum ); + auto branch = fork_db.fetch_branch( fork_head->id, fork_head->dpos_irreversible_blocknum ); try { const auto& rbi = reversible_blocks.get_index(); + std::vector>>> v; + v.reserve( branch.size() ); + for( auto bitr = branch.rbegin(); bitr != branch.rend(); ++bitr ) { + v.emplace_back( blog.create_append_future( thread_pool.get_executor(), (*bitr)->block, + packed_transaction::cf_compression_type::none ) ); + } + auto it = v.begin(); + for( auto bitr = branch.rbegin(); bitr != branch.rend(); ++bitr ) { if( read_mode == db_read_mode::IRREVERSIBLE ) { apply_block( *bitr, controller::block_status::complete, trx_meta_cache_lookup{} ); @@ -418,10 +360,13 @@ struct controller_impl { emit( self.irreversible_block, *bitr ); - db.commit( (*bitr)->block_num ); - root_id = (*bitr)->id; + // blog.append could fail due to failures like running out of space. + // Do it before commit so that in case it throws, DB can be rolled back. + blog.append( std::move( *it ) ); + ++it; - blog.append( (*bitr)->block ); + kv_db.commit( (*bitr)->block_num ); + root_id = (*bitr)->id; auto rbitr = rbi.begin(); while( rbitr != rbi.end() && rbitr->blocknum <= (*bitr)->block_num ) { @@ -429,18 +374,20 @@ struct controller_impl { rbitr = rbi.begin(); } } - } catch( fc::exception& ) { + } catch( std::exception& ) { if( root_id != fork_db.root()->id ) { fork_db.advance_root( root_id ); } throw; } - //db.commit( fork_head->dpos_irreversible_blocknum ); // redundant - if( root_id != fork_db.root()->id ) { + branch.emplace_back(fork_db.root()); fork_db.advance_root( root_id ); } + + // delete branch in thread pool + boost::asio::post( thread_pool.get_executor(), [branch{std::move(branch)}]() {} ); } /** @@ -458,18 +405,18 @@ struct controller_impl { genheader.pending_schedule.schedule_hash = fc::sha256::hash(initial_legacy_schedule); genheader.header.timestamp = genesis.initial_timestamp; genheader.header.action_mroot = genesis.compute_chain_id(); - genheader.id = genheader.header.id(); + genheader.id = genheader.header.calculate_id(); genheader.block_num = genheader.header.block_num(); head = std::make_shared(); static_cast(*head) = genheader; head->activated_protocol_features = std::make_shared(); head->block = std::make_shared(genheader.header); - db.set_revision( head->block_num ); + kv_db.set_revision( head->block_num ); initialize_database(genesis); } - void replay(std::function shutdown) { + void replay(std::function check_shutdown) { auto blog_head = blog.head(); auto blog_head_time = blog_head->timestamp.to_time_point(); replay_head_time = blog_head_time; @@ -482,11 +429,12 @@ struct controller_impl { ilog( "existing block log, attempting to replay from ${s} to ${n} blocks", ("s", start_block_num)("n", blog_head->block_num()) ); try { - while( auto next = blog.read_block_by_num( head->block_num + 1 ) ) { - replay_push_block( next, controller::block_status::irreversible ); - if( next->block_num() % 500 == 0 ) { - ilog( "${n} of ${head}", ("n", next->block_num())("head", blog_head->block_num()) ); - if( shutdown() ) break; + while( std::unique_ptr next = blog.read_signed_block_by_num( head->block_num + 1 ) ) { + auto block_num = next->block_num(); + replay_push_block( std::move(next), controller::block_status::irreversible ); + if( check_shutdown() ) break; + if( block_num % 500 == 0 ) { + ilog( "${n} of ${head}", ("n", block_num)("head", blog_head->block_num()) ); } } } catch( const database_guard_exception& e ) { @@ -511,14 +459,15 @@ struct controller_impl { // if the irreverible log is played without undo sessions enabled, we need to sync the // revision ordinal to the appropriate expected value here. if( self.skip_db_sessions( controller::block_status::irreversible ) ) - db.set_revision( head->block_num ); + kv_db.set_revision( head->block_num ); } else { ilog( "no irreversible blocks need to be replayed" ); } - if( !except_ptr && !shutdown() ) { + if( !except_ptr && !check_shutdown() ) { int rev = 0; while( auto obj = reversible_blocks.find(head->block_num+1) ) { + if( check_shutdown() ) break; ++rev; replay_push_block( obj->get_block(), controller::block_status::validated ); } @@ -536,15 +485,22 @@ struct controller_impl { } } - void startup(std::function shutdown, const snapshot_reader_ptr& snapshot) { + void startup(std::function shutdown, std::function check_shutdown, const snapshot_reader_ptr& snapshot) { EOS_ASSERT( snapshot, snapshot_exception, "No snapshot reader provided" ); + EOS_ASSERT( db.revision() == kv_db.revision(), database_revision_mismatch_exception, + "chainbase is at revision ${a}, but chain-kv is at revision ${b}", ("a", db.revision())("b", kv_db.revision()) ); + this->shutdown = shutdown; ilog( "Starting initialization from snapshot, this may take a significant amount of time" ); try { snapshot->validate(); if( blog.head() ) { - read_from_snapshot( snapshot, blog.first_block_num(), blog.head()->block_num() ); + kv_db.read_from_snapshot( snapshot, blog.first_block_num(), blog.head()->block_num(), + authorization, resource_limits, + fork_db, head, snapshot_head_block, chain_id ); } else { - read_from_snapshot( snapshot, 0, std::numeric_limits::max() ); + kv_db.read_from_snapshot( snapshot, 0, std::numeric_limits::max(), + authorization, resource_limits, + fork_db, head, snapshot_head_block, chain_id ); const uint32_t lib_num = head->block_num; EOS_ASSERT( lib_num > 0, snapshot_exception, "Snapshot indicates controller head at block number 0, but that is not allowed. " @@ -554,7 +510,7 @@ struct controller_impl { const auto hash = calculate_integrity_hash(); ilog( "database initialized with hash: ${hash}", ("hash", hash) ); - init(shutdown); + init(check_shutdown, true); } catch (boost::interprocess::bad_alloc& e) { elog( "db storage not configured to have enough storage for the provided snapshot, please increase and retry snapshot" ); throw e; @@ -563,14 +519,17 @@ struct controller_impl { ilog( "Finished initialization from snapshot" ); } - void startup(std::function shutdown, const genesis_state& genesis) { + void startup(std::function shutdown, std::function check_shutdown, const genesis_state& genesis) { EOS_ASSERT( db.revision() < 1, database_exception, "This version of controller::startup only works with a fresh state database." ); + EOS_ASSERT( db.revision() == kv_db.revision(), database_revision_mismatch_exception, + "chainbase is at revision ${a}, but chain-kv is at revision ${b}", ("a", db.revision())("b", kv_db.revision()) ); const auto& genesis_chain_id = genesis.compute_chain_id(); EOS_ASSERT( genesis_chain_id == chain_id, chain_id_type_exception, "genesis state provided to startup corresponds to a chain ID (${genesis_chain_id}) that does not match the chain ID that controller was constructed with (${controller_chain_id})", ("genesis_chain_id", genesis_chain_id)("controller_chain_id", chain_id) ); + this->shutdown = shutdown; if( fork_db.head() ) { if( read_mode == db_read_mode::IRREVERSIBLE && fork_db.head()->id != fork_db.root()->id ) { fork_db.rollback_head_to_root(); @@ -590,15 +549,18 @@ struct controller_impl { "block log does not start with genesis block" ); } else { - blog.reset( genesis, head->block ); + blog.reset( genesis, head->block, packed_transaction::cf_compression_type::none ); } - init(shutdown); + init(check_shutdown, true); } - void startup(std::function shutdown) { + void startup(std::function shutdown, std::function check_shutdown) { EOS_ASSERT( db.revision() >= 1, database_exception, "This version of controller::startup does not work with a fresh state database." ); + EOS_ASSERT( db.revision() == kv_db.revision(), database_revision_mismatch_exception, + "chainbase is at revision ${a}, but chain-kv is at revision ${b}", ("a", db.revision())("b", kv_db.revision()) ); EOS_ASSERT( fork_db.head(), fork_database_exception, "No existing fork database despite existing chain state. Replay required." ); + this->shutdown = shutdown; uint32_t lib_num = fork_db.root()->block_num; auto first_block_num = blog.first_block_num(); if( blog.head() ) { @@ -621,7 +583,7 @@ struct controller_impl { } head = fork_db.head(); - init(shutdown); + init(check_shutdown, false); } @@ -638,7 +600,7 @@ struct controller_impl { return header_itr; } - void init(std::function shutdown) { + void init(std::function check_shutdown, bool clean_startup) { uint32_t lib_num = (blog.head() ? blog.head()->block_num() : fork_db.root()->block_num); auto header_itr = validate_db_version( db ); @@ -658,6 +620,8 @@ struct controller_impl { }); } + kv_db.check_backing_store_setting( clean_startup ); + // At this point head != nullptr && fork_db.head() != nullptr && fork_db.root() != nullptr. // Furthermore, fork_db.root()->block_num <= lib_num. // Also, even though blog.head() may still be nullptr, blog.first_block_num() is guaranteed to be lib_num + 1. @@ -672,7 +636,7 @@ struct controller_impl { ("db",db.revision())("head",head->block_num) ); } while( db.revision() > head->block_num ) { - db.undo(); + kv_db.undo(); } protocol_features.init( db ); @@ -738,11 +702,31 @@ struct controller_impl { // else no checks needed since fork_db will be completely reset on replay anyway } + if (auto dm_logger = get_deep_mind_logger()) { + // FIXME: We should probably feed that from CMake directly somehow ... + fc_dlog(*dm_logger, "DEEP_MIND_VERSION 13 0"); + + fc_dlog(*dm_logger, "ABIDUMP START ${block_num} ${global_sequence_num}", + ("block_num", head->block_num) + ("global_sequence_num", db.get().global_action_sequence) + ); + const auto& idx = db.get_index(); + for (auto& row : idx.indices()) { + if (row.abi.size() != 0) { + fc_dlog(*dm_logger, "ABIDUMP ABI ${contract} ${abi}", + ("contract", row.name) + ("abi", row.abi) + ); + } + } + fc_dlog(*dm_logger, "ABIDUMP END"); + } + if( last_block_num > head->block_num ) { - replay( shutdown ); // replay any irreversible and reversible blocks ahead of current head + replay( check_shutdown ); // replay any irreversible and reversible blocks ahead of current head } - if( shutdown() ) return; + if( check_shutdown() ) return; if( read_mode != db_read_mode::IRREVERSIBLE && fork_db.pending_head()->id != fork_db.head()->id @@ -769,228 +753,17 @@ struct controller_impl { reversible_blocks.add_index(); controller_index_set::add_indices(db); + db.add_index(); contract_database_index_set::add_indices(db); authorization.add_indices(); resource_limits.add_indices(); } - void clear_all_undo() { - // Rewind the database to the last irreversible block - db.undo_all(); - /* - FC_ASSERT(db.revision() == self.head_block_num(), - "Chainbase revision does not match head block num", - ("rev", db.revision())("head_block", self.head_block_num())); - */ - } - - void add_contract_tables_to_snapshot( const snapshot_writer_ptr& snapshot ) const { - snapshot->write_section("contract_tables", [this]( auto& section ) { - index_utils::walk(db, [this, §ion]( const table_id_object& table_row ){ - // add a row for the table - section.add_row(table_row, db); - - // followed by a size row and then N data rows for each type of table - contract_database_index_set::walk_indices([this, §ion, &table_row]( auto utils ) { - using utils_t = decltype(utils); - using value_t = typename decltype(utils)::index_t::value_type; - using by_table_id = object_to_table_id_tag_t; - - auto tid_key = boost::make_tuple(table_row.id); - auto next_tid_key = boost::make_tuple(table_id_object::id_type(table_row.id._id + 1)); - - unsigned_int size = utils_t::template size_range(db, tid_key, next_tid_key); - section.add_row(size, db); - - utils_t::template walk_range(db, tid_key, next_tid_key, [this, §ion]( const auto &row ) { - section.add_row(row, db); - }); - }); - }); - }); - } - - void read_contract_tables_from_snapshot( const snapshot_reader_ptr& snapshot ) { - snapshot->read_section("contract_tables", [this]( auto& section ) { - bool more = !section.empty(); - while (more) { - // read the row for the table - table_id_object::id_type t_id; - index_utils::create(db, [this, §ion, &t_id](auto& row) { - section.read_row(row, db); - t_id = row.id; - }); - - // read the size and data rows for each type of table - contract_database_index_set::walk_indices([this, §ion, &t_id, &more](auto utils) { - using utils_t = decltype(utils); - - unsigned_int size; - more = section.read_row(size, db); - - for (size_t idx = 0; idx < size.value; idx++) { - utils_t::create(db, [this, §ion, &more, &t_id](auto& row) { - row.t_id = t_id; - more = section.read_row(row, db); - }); - } - }); - } - }); - } - - void add_to_snapshot( const snapshot_writer_ptr& snapshot ) const { - snapshot->write_section([this]( auto §ion ){ - section.add_row(chain_snapshot_header(), db); - }); - - snapshot->write_section([this]( auto §ion ){ - section.template add_row(*fork_db.head(), db); - }); - - controller_index_set::walk_indices([this, &snapshot]( auto utils ){ - using value_t = typename decltype(utils)::index_t::value_type; - - // skip the table_id_object as its inlined with contract tables section - if (std::is_same::value) { - return; - } - - // skip the database_header as it is only relevant to in-memory database - if (std::is_same::value) { - return; - } - - snapshot->write_section([this]( auto& section ){ - decltype(utils)::walk(db, [this, §ion]( const auto &row ) { - section.add_row(row, db); - }); - }); - }); - - add_contract_tables_to_snapshot(snapshot); - - authorization.add_to_snapshot(snapshot); - resource_limits.add_to_snapshot(snapshot); - } - - static fc::optional extract_legacy_genesis_state( snapshot_reader& snapshot, uint32_t version ) { - fc::optional genesis; - using v2 = legacy::snapshot_global_property_object_v2; - - if (std::clamp(version, v2::minimum_version, v2::maximum_version) == version ) { - genesis.emplace(); - snapshot.read_section([&genesis=*genesis]( auto §ion ){ - section.read_row(genesis); - }); - } - return genesis; - } - - void read_from_snapshot( const snapshot_reader_ptr& snapshot, uint32_t blog_start, uint32_t blog_end ) { - chain_snapshot_header header; - snapshot->read_section([this, &header]( auto §ion ){ - section.read_row(header, db); - header.validate(); - }); - - { /// load and upgrade the block header state - block_header_state head_header_state; - using v2 = legacy::snapshot_block_header_state_v2; - - if (std::clamp(header.version, v2::minimum_version, v2::maximum_version) == header.version ) { - snapshot->read_section([this, &head_header_state]( auto §ion ) { - legacy::snapshot_block_header_state_v2 legacy_header_state; - section.read_row(legacy_header_state, db); - head_header_state = block_header_state(std::move(legacy_header_state)); - }); - } else { - snapshot->read_section([this,&head_header_state]( auto §ion ){ - section.read_row(head_header_state, db); - }); - } - - snapshot_head_block = head_header_state.block_num; - EOS_ASSERT( blog_start <= (snapshot_head_block + 1) && snapshot_head_block <= blog_end, - block_log_exception, - "Block log is provided with snapshot but does not contain the head block from the snapshot nor a block right after it", - ("snapshot_head_block", snapshot_head_block) - ("block_log_first_num", blog_start) - ("block_log_last_num", blog_end) - ); - - fork_db.reset( head_header_state ); - head = fork_db.head(); - snapshot_head_block = head->block_num; - - } - - controller_index_set::walk_indices([this, &snapshot, &header]( auto utils ){ - using value_t = typename decltype(utils)::index_t::value_type; - - // skip the table_id_object as its inlined with contract tables section - if (std::is_same::value) { - return; - } - - // skip the database_header as it is only relevant to in-memory database - if (std::is_same::value) { - return; - } - - // special case for in-place upgrade of global_property_object - if (std::is_same::value) { - using v2 = legacy::snapshot_global_property_object_v2; - - if (std::clamp(header.version, v2::minimum_version, v2::maximum_version) == header.version ) { - fc::optional genesis = extract_legacy_genesis_state(*snapshot, header.version); - EOS_ASSERT( genesis, snapshot_exception, - "Snapshot indicates chain_snapshot_header version 2, but does not contain a genesis_state. " - "It must be corrupted."); - snapshot->read_section([&db=this->db,gs_chain_id=genesis->compute_chain_id()]( auto §ion ) { - v2 legacy_global_properties; - section.read_row(legacy_global_properties, db); - - db.create([&legacy_global_properties,&gs_chain_id](auto& gpo ){ - gpo.initalize_from(legacy_global_properties, gs_chain_id); - }); - }); - return; // early out to avoid default processing - } - } - - snapshot->read_section([this]( auto& section ) { - bool more = !section.empty(); - while(more) { - decltype(utils)::create(db, [this, §ion, &more]( auto &row ) { - more = section.read_row(row, db); - }); - } - }); - }); - - read_contract_tables_from_snapshot(snapshot); - - authorization.read_from_snapshot(snapshot); - resource_limits.read_from_snapshot(snapshot); - - db.set_revision( head->block_num ); - db.create([](const auto& header){ - // nothing to do - }); - - const auto& gpo = db.get(); - EOS_ASSERT( gpo.chain_id == chain_id, chain_id_type_exception, - "chain ID in snapshot (${snapshot_chain_id}) does not match the chain ID that controller was constructed with (${controller_chain_id})", - ("snapshot_chain_id", gpo.chain_id)("controller_chain_id", chain_id) - ); - } - sha256 calculate_integrity_hash() const { sha256::encoder enc; auto hash_writer = std::make_shared(enc); - add_to_snapshot(hash_writer); + kv_db.add_to_snapshot(hash_writer, *fork_db.head(), authorization, resource_limits); hash_writer->finalize(); return enc.result(); @@ -1004,8 +777,7 @@ struct controller_impl { if( name == config::system_account_name ) { // The initial eosio ABI value affects consensus; see https://github.com/EOSIO/eos/issues/7794 // TODO: This doesn't charge RAM; a fix requires a consensus upgrade. - a.abi.resize(sizeof(eosio_abi_bin)); - memcpy(a.abi.data(), eosio_abi_bin, sizeof(eosio_abi_bin)); + a.abi.assign(eosio_abi_bin, sizeof(eosio_abi_bin)); } }); db.create([&](auto & a) { @@ -1014,9 +786,9 @@ struct controller_impl { }); const auto& owner_permission = authorization.create_permission(name, config::owner_name, 0, - owner, initial_timestamp ); + owner, 0, initial_timestamp ); const auto& active_permission = authorization.create_permission(name, config::active_name, owner_permission.id, - active, initial_timestamp ); + active, 0, initial_timestamp ); resource_limits.initialize_account(name); @@ -1025,7 +797,12 @@ struct controller_impl { ram_delta += owner_permission.auth.get_billable_size(); ram_delta += active_permission.auth.get_billable_size(); - resource_limits.add_pending_ram_usage(name, ram_delta); + std::string event_id; + if (get_deep_mind_logger() != nullptr) { + event_id = STORAGE_EVENT_ID("${name}", ("name", name)); + } + + resource_limits.add_pending_ram_usage(name, ram_delta, storage_usage_trace(0, std::move(event_id), "account", "add", "newaccount")); resource_limits.verify_account_ram_usage(name); } @@ -1047,6 +824,9 @@ struct controller_impl { genesis.initial_configuration.validate(); db.create([&genesis,&chain_id=this->chain_id](auto& gpo ){ gpo.configuration = genesis.initial_configuration; + gpo.kv_configuration = kv_database_config{}; + // TODO: Update this when genesis protocol features are enabled. + gpo.wasm_configuration = genesis_state::default_initial_wasm_configuration; gpo.chain_id = chain_id; }); @@ -1058,6 +838,7 @@ struct controller_impl { }); db.create([](auto&){}); + db.create([](auto&){}); authorization.initialize_database(); resource_limits.initialize_database(); @@ -1076,30 +857,37 @@ struct controller_impl { config::majority_producers_permission_name, active_permission.id, active_producers_authority, + 0, genesis.initial_timestamp ); const auto& minority_permission = authorization.create_permission( config::producers_account_name, config::minority_producers_permission_name, majority_permission.id, active_producers_authority, + 0, genesis.initial_timestamp ); } // The returned scoped_exit should not exceed the lifetime of the pending which existed when make_block_restore_point was called. fc::scoped_exit> make_block_restore_point() { - auto& bb = pending->_block_stage.get(); - auto orig_block_transactions_size = bb._pending_trx_receipts.size(); - auto orig_state_transactions_size = bb._pending_trx_metas.size(); - auto orig_state_actions_size = bb._actions.size(); + auto& bb = std::get(pending->_block_stage); + auto orig_trx_receipts_size = bb._pending_trx_receipts.size(); + auto orig_trx_metas_size = bb._pending_trx_metas.size(); + auto orig_trx_receipt_digests_size = std::holds_alternative(bb._trx_mroot_or_receipt_digests) ? + std::get(bb._trx_mroot_or_receipt_digests).size() : 0; + auto orig_action_receipt_digests_size = bb._action_receipt_digests.size(); std::function callback = [this, - orig_block_transactions_size, - orig_state_transactions_size, - orig_state_actions_size]() + orig_trx_receipts_size, + orig_trx_metas_size, + orig_trx_receipt_digests_size, + orig_action_receipt_digests_size]() { - auto& bb = pending->_block_stage.get(); - bb._pending_trx_receipts.resize(orig_block_transactions_size); - bb._pending_trx_metas.resize(orig_state_transactions_size); - bb._actions.resize(orig_state_actions_size); + auto& bb = std::get(pending->_block_stage); + bb._pending_trx_receipts.resize(orig_trx_receipts_size); + bb._pending_trx_metas.resize(orig_trx_metas_size); + if( std::holds_alternative(bb._trx_mroot_or_receipt_digests) ) + std::get(bb._trx_mroot_or_receipt_digests).resize(orig_trx_receipt_digests_size); + bb._action_receipt_digests.resize(orig_action_receipt_digests_size); }; return fc::make_scoped_exit( std::move(callback) ); @@ -1127,43 +915,70 @@ struct controller_impl { etrx.set_reference_block( self.head_block_id() ); } + if (auto dm_logger = get_deep_mind_logger()) { + auto packed_trx = fc::raw::pack(etrx); + + fc_dlog(*dm_logger, "TRX_OP CREATE onerror ${id} ${trx}", + ("id", etrx.id()) + ("trx", fc::to_hex(packed_trx)) + ); + } + transaction_checktime_timer trx_timer(timer); - transaction_context trx_context( self, etrx, etrx.id(), std::move(trx_timer), start ); + const packed_transaction trx( std::move( etrx ), true ); + transaction_context trx_context( self, trx, std::move(trx_timer), start ); trx_context.deadline = deadline; trx_context.explicit_billed_cpu_time = explicit_billed_cpu_time; trx_context.billed_cpu_time_us = billed_cpu_time_us; trx_context.enforce_whiteblacklist = enforce_whiteblacklist; transaction_trace_ptr trace = trx_context.trace; + + auto handle_exception = [&](const auto& e) + { + cpu_time_to_bill_us = trx_context.update_billed_cpu_time( fc::time_point::now() ); + trace->error_code = controller::convert_exception_to_error_code( e ); + trace->except = e; + trace->except_ptr = std::current_exception(); + }; + try { trx_context.init_for_implicit_trx(); trx_context.published = gtrx.published; - trx_context.execute_action( trx_context.schedule_action( etrx.actions.back(), gtrx.sender, false, 0, 0 ), 0 ); + trx_context.execute_action( trx_context.schedule_action( trx.get_transaction().actions.back(), gtrx.sender, false, 0, 0 ), 0 ); trx_context.finalize(); // Automatically rounds up network and CPU usage in trace and bills payers if successful auto restore = make_block_restore_point(); trace->receipt = push_receipt( gtrx.trx_id, transaction_receipt::soft_fail, trx_context.billed_cpu_time_us, trace->net_usage ); - fc::move_append( pending->_block_stage.get()._actions, move(trx_context.executed) ); + fc::move_append( std::get(pending->_block_stage)._action_receipt_digests, + std::move(trx_context.executed_action_receipt_digests) ); trx_context.squash(); restore.cancel(); return trace; - } catch( const disallowed_transaction_extensions_bad_block_exception& ) { + } catch( const objective_block_validation_exception& ) { + throw; + } catch ( const std::bad_alloc& ) { throw; - } catch( const protocol_feature_bad_block_exception& ) { + } catch ( const boost::interprocess::bad_alloc& ) { throw; } catch( const fc::exception& e ) { - cpu_time_to_bill_us = trx_context.update_billed_cpu_time( fc::time_point::now() ); - trace->error_code = controller::convert_exception_to_error_code( e ); - trace->except = e; - trace->except_ptr = std::current_exception(); + handle_exception(e); + } catch ( const std::exception& e ) { + auto wrapper = fc::std_exception_wrapper::from_current_exception(e); + handle_exception(wrapper); } return trace; } int64_t remove_scheduled_transaction( const generated_transaction_object& gto ) { + std::string event_id; + if (get_deep_mind_logger() != nullptr) { + event_id = STORAGE_EVENT_ID("${id}", ("id", gto.id)); + } + int64_t ram_delta = -(config::billable_size_v + gto.packed_trx.size()); - resource_limits.add_pending_ram_usage( gto.payer, ram_delta ); + resource_limits.add_pending_ram_usage( gto.payer, ram_delta, storage_usage_trace(0, std::move(event_id), "deferred_trx", "remove", "deferred_trx_removed") ); // No need to verify_account_ram_usage since we are only reducing memory db.remove( gto ); @@ -1208,9 +1023,7 @@ struct controller_impl { const bool validating = !self.is_producing_block(); EOS_ASSERT( !validating || explicit_billed_cpu_time, transaction_exception, "validating requires explicit billing" ); - maybe_session undo_session; - if ( !self.skip_db_sessions() ) - undo_session = maybe_session(db); + combined_session undo_session = !self.skip_db_sessions() ? kv_db.make_session() : kv_db.make_no_op_session(); auto gtrx = generated_transaction(gto); @@ -1229,7 +1042,9 @@ struct controller_impl { signed_transaction dtrx; fc::raw::unpack(ds,static_cast(dtrx) ); - transaction_metadata_ptr trx = transaction_metadata::create_no_recover_keys( packed_transaction( dtrx ), transaction_metadata::trx_type::scheduled ); + transaction_metadata_ptr trx = + transaction_metadata::create_no_recover_keys( std::make_shared( std::move(dtrx), true ), + transaction_metadata::trx_type::scheduled ); trx->accepted = true; transaction_trace_ptr trace; @@ -1243,7 +1058,7 @@ struct controller_impl { trace->receipt = push_receipt( gtrx.trx_id, transaction_receipt::expired, billed_cpu_time_us, 0 ); // expire the transaction trace->account_ram_delta = account_delta( gtrx.payer, trx_removal_ram_delta ); emit( self.accepted_transaction, trx ); - emit( self.applied_transaction, std::tie(trace, dtrx) ); + emit( self.applied_transaction, std::tie(trace, trx->packed_trx()) ); undo_session.squash(); return trace; } @@ -1256,19 +1071,35 @@ struct controller_impl { uint32_t cpu_time_to_bill_us = billed_cpu_time_us; transaction_checktime_timer trx_timer(timer); - transaction_context trx_context( self, dtrx, gtrx.trx_id, std::move(trx_timer) ); + transaction_context trx_context( self, *trx->packed_trx(), std::move(trx_timer) ); trx_context.leeway = fc::microseconds(0); // avoid stealing cpu resource trx_context.deadline = deadline; trx_context.explicit_billed_cpu_time = explicit_billed_cpu_time; trx_context.billed_cpu_time_us = billed_cpu_time_us; trx_context.enforce_whiteblacklist = gtrx.sender.empty() ? true : !sender_avoids_whitelist_blacklist_enforcement( gtrx.sender ); trace = trx_context.trace; + + auto handle_exception = [&](const auto& e) + { + cpu_time_to_bill_us = trx_context.update_billed_cpu_time( fc::time_point::now() ); + trace->error_code = controller::convert_exception_to_error_code( e ); + trace->except = e; + trace->except_ptr = std::current_exception(); + trace->elapsed = fc::time_point::now() - trx_context.start; + + if (auto dm_logger = get_deep_mind_logger()) { + fc_dlog(*dm_logger, "DTRX_OP FAILED ${action_id}", + ("action_id", trx_context.get_action_id()) + ); + } + }; + try { trx_context.init_for_deferred_trx( gtrx.published ); if( trx_context.enforce_whiteblacklist && pending->_block_status == controller::block_status::incomplete ) { flat_set actors; - for( const auto& act : trx_context.trx.actions ) { + for( const auto& act : trx->packed_trx()->get_transaction().actions ) { for( const auto& auth : act.authorization ) { actors.insert( auth.actor ); } @@ -1286,12 +1117,13 @@ struct controller_impl { trx_context.billed_cpu_time_us, trace->net_usage ); - fc::move_append( pending->_block_stage.get()._actions, move(trx_context.executed) ); + fc::move_append( std::get(pending->_block_stage)._action_receipt_digests, + std::move(trx_context.executed_action_receipt_digests) ); trace->account_ram_delta = account_delta( gtrx.payer, trx_removal_ram_delta ); emit( self.accepted_transaction, trx ); - emit( self.applied_transaction, std::tie(trace, dtrx) ); + emit( self.applied_transaction, std::tie(trace, trx->packed_trx()) ); trx_context.squash(); undo_session.squash(); @@ -1299,17 +1131,19 @@ struct controller_impl { restore.cancel(); return trace; - } catch( const disallowed_transaction_extensions_bad_block_exception& ) { + } catch( const objective_block_validation_exception& ) { throw; - } catch( const protocol_feature_bad_block_exception& ) { + } catch ( const std::bad_alloc& ) { + throw; + } catch ( const boost::interprocess::bad_alloc& ) { throw; } catch( const fc::exception& e ) { - cpu_time_to_bill_us = trx_context.update_billed_cpu_time( fc::time_point::now() ); - trace->error_code = controller::convert_exception_to_error_code( e ); - trace->except = e; - trace->except_ptr = std::current_exception(); - trace->elapsed = fc::time_point::now() - trx_context.start; + handle_exception(e); + } catch ( const std::exception& e) { + auto wrapper = fc::std_exception_wrapper::from_current_exception(e); + handle_exception(wrapper); } + trx_context.undo(); // Only subjective OR soft OR hard failure logic below: @@ -1325,7 +1159,7 @@ struct controller_impl { if( !trace->except_ptr ) { trace->account_ram_delta = account_delta( gtrx.payer, trx_removal_ram_delta ); emit( self.accepted_transaction, trx ); - emit( self.applied_transaction, std::tie(trace, dtrx) ); + emit( self.applied_transaction, std::tie(trace, trx->packed_trx()) ); undo_session.squash(); return trace; } @@ -1366,12 +1200,12 @@ struct controller_impl { trace->account_ram_delta = account_delta( gtrx.payer, trx_removal_ram_delta ); emit( self.accepted_transaction, trx ); - emit( self.applied_transaction, std::tie(trace, dtrx) ); + emit( self.applied_transaction, std::tie(trace, trx->packed_trx()) ); undo_session.squash(); } else { emit( self.accepted_transaction, trx ); - emit( self.applied_transaction, std::tie(trace, dtrx) ); + emit( self.applied_transaction, std::tie(trace, trx->packed_trx()) ); } return trace; @@ -1386,12 +1220,15 @@ struct controller_impl { uint64_t cpu_usage_us, uint64_t net_usage ) { uint64_t net_usage_words = net_usage / 8; EOS_ASSERT( net_usage_words*8 == net_usage, transaction_exception, "net_usage is not divisible by 8" ); - auto& receipts = pending->_block_stage.get()._pending_trx_receipts; + auto& receipts = std::get(pending->_block_stage)._pending_trx_receipts; receipts.emplace_back( trx ); transaction_receipt& r = receipts.back(); r.cpu_usage_us = cpu_usage_us; r.net_usage_words = net_usage_words; r.status = status; + auto& bb = std::get(pending->_block_stage); + if( std::holds_alternative(bb._trx_mroot_or_receipt_digests) ) + std::get(bb._trx_mroot_or_receipt_digests).emplace_back( r.digest() ); return r; } @@ -1404,6 +1241,7 @@ struct controller_impl { fc::time_point deadline, uint32_t billed_cpu_time_us, bool explicit_billed_cpu_time, + std::optional explicit_net_usage_words, uint32_t subjective_cpu_bill_us ) { EOS_ASSERT(deadline != fc::time_point(), transaction_exception, "deadline cannot be uninitialized"); @@ -1424,9 +1262,8 @@ struct controller_impl { } } - const signed_transaction& trn = trx->packed_trx()->get_signed_transaction(); transaction_checktime_timer trx_timer(timer); - transaction_context trx_context(self, trn, trx->id(), std::move(trx_timer), start); + transaction_context trx_context(self, *trx->packed_trx(), std::move(trx_timer), start); if ((bool)subjective_cpu_leeway && pending->_block_status == controller::block_status::incomplete) { trx_context.leeway = *subjective_cpu_leeway; } @@ -1435,15 +1272,30 @@ struct controller_impl { trx_context.billed_cpu_time_us = billed_cpu_time_us; trx_context.subjective_cpu_bill_us = subjective_cpu_bill_us; trace = trx_context.trace; + + auto handle_exception =[&](const auto& e) + { + trace->error_code = controller::convert_exception_to_error_code( e ); + trace->except = e; + trace->except_ptr = std::current_exception(); + trace->elapsed = fc::time_point::now() - trx_context.start; + }; + try { + const transaction& trn = trx->packed_trx()->get_transaction(); if( trx->implicit ) { + EOS_ASSERT( !explicit_net_usage_words, transaction_exception, "NET usage cannot be explicitly set for implicit transactions" ); trx_context.init_for_implicit_trx(); trx_context.enforce_whiteblacklist = false; } else { bool skip_recording = replay_head_time && (time_point(trn.expiration) <= *replay_head_time); - trx_context.init_for_input_trx( trx->packed_trx()->get_unprunable_size(), - trx->packed_trx()->get_prunable_size(), - skip_recording); + if( explicit_net_usage_words ) { + trx_context.init_for_input_trx_with_explicit_net( *explicit_net_usage_words, skip_recording ); + } else { + trx_context.init_for_input_trx( trx->packed_trx()->get_unprunable_size(), + trx->packed_trx()->get_prunable_size(), + skip_recording ); + } } trx_context.delay = fc::seconds(trn.delay_sec); @@ -1469,7 +1321,7 @@ struct controller_impl { : transaction_receipt::delayed; trace->receipt = push_receipt(*trx->packed_trx(), s, trx_context.billed_cpu_time_us, trace->net_usage); trx->billed_cpu_time_us = trx_context.billed_cpu_time_us; - pending->_block_stage.get()._pending_trx_metas.emplace_back(trx); + std::get(pending->_block_stage)._pending_trx_metas.emplace_back(trx); } else { transaction_receipt_header r; r.status = transaction_receipt::executed; @@ -1478,7 +1330,8 @@ struct controller_impl { trace->receipt = r; } - fc::move_append(pending->_block_stage.get()._actions, move(trx_context.executed)); + fc::move_append( std::get(pending->_block_stage)._action_receipt_digests, + std::move(trx_context.executed_action_receipt_digests) ); // call the accept signal but only once for this transaction if (!trx->accepted) { @@ -1486,7 +1339,7 @@ struct controller_impl { emit( self.accepted_transaction, trx); } - emit(self.applied_transaction, std::tie(trace, trn)); + emit(self.applied_transaction, std::tie(trace, trx->packed_trx())); if ( read_mode != db_read_mode::SPECULATIVE && pending->_block_status == controller::block_status::incomplete ) { @@ -1498,19 +1351,21 @@ struct controller_impl { } return trace; - } catch( const disallowed_transaction_extensions_bad_block_exception& ) { - throw; - } catch( const protocol_feature_bad_block_exception& ) { + } catch( const objective_block_validation_exception& ) { throw; + } catch ( const std::bad_alloc& ) { + throw; + } catch ( const boost::interprocess::bad_alloc& ) { + throw; } catch (const fc::exception& e) { - trace->error_code = controller::convert_exception_to_error_code( e ); - trace->except = e; - trace->except_ptr = std::current_exception(); - trace->elapsed = fc::time_point::now() - trx_context.start; + handle_exception(e); + } catch (const std::exception& e) { + auto wrapper = fc::std_exception_wrapper::from_current_exception(e); + handle_exception(wrapper); } emit( self.accepted_transaction, trx ); - emit( self.applied_transaction, std::tie(trace, trn) ); + emit( self.applied_transaction, std::tie(trace, trx->packed_trx()) ); return trace; } FC_CAPTURE_AND_RETHROW((trace)) @@ -1520,10 +1375,15 @@ struct controller_impl { uint16_t confirm_block_count, const vector& new_protocol_feature_activations, controller::block_status s, - const optional& producer_block_id ) + const std::optional& producer_block_id ) { EOS_ASSERT( !pending, block_validate_exception, "pending block already exists" ); + if (auto dm_logger = get_deep_mind_logger()) { + // The head block represents the block just before this one that is about to start, so add 1 to get this block num + fc_dlog(*dm_logger, "START_BLOCK ${block_num}", ("block_num", head->block_num + 1)); + } + emit( self.block_start, head->block_num + 1 ); auto guard_pending = fc::make_scoped_exit([this, head_block_num=head->block_num](){ @@ -1535,18 +1395,17 @@ struct controller_impl { EOS_ASSERT( db.revision() == head->block_num, database_exception, "db revision is not on par with head block", ("db.revision()", db.revision())("controller_head_block", head->block_num)("fork_db_head_block", fork_db.head()->block_num) ); - pending.emplace( maybe_session(db), *head, when, confirm_block_count, new_protocol_feature_activations ); + pending.emplace( kv_db.make_session(), *head, when, confirm_block_count, new_protocol_feature_activations ); } else { - pending.emplace( maybe_session(), *head, when, confirm_block_count, new_protocol_feature_activations ); + pending.emplace( kv_db.make_no_op_session(), *head, when, confirm_block_count, new_protocol_feature_activations ); } pending->_block_status = s; pending->_producer_block_id = producer_block_id; - auto& bb = pending->_block_stage.get(); + auto& bb = std::get(pending->_block_stage); const auto& pbhs = bb._pending_block_header_state; - // modify state of speculative block only if we are in speculative read mode (otherwise we need clean state for head or read-only modes) if ( read_mode == db_read_mode::SPECULATIVE || pending->_block_status != controller::block_status::incomplete ) { const auto& pso = db.get(); @@ -1615,9 +1474,9 @@ struct controller_impl { }); } - const auto& gpo = db.get(); + const auto& gpo = self.get_global_properties(); - if( gpo.proposed_schedule_block_num.valid() && // if there is a proposed schedule that was proposed in a block ... + if( gpo.proposed_schedule_block_num && // if there is a proposed schedule that was proposed in a block ... ( *gpo.proposed_schedule_block_num <= pbhs.dpos_irreversible_blocknum ) && // ... that has now become irreversible ... pbhs.prev_pending_schedule.schedule.producers.size() == 0 // ... and there was room for a new pending schedule prior to any possible promotion ) @@ -1633,9 +1492,9 @@ struct controller_impl { EOS_ASSERT( gpo.proposed_schedule.version == pbhs.active_schedule_version + 1, producer_schedule_exception, "wrong producer schedule version specified" ); - pending->_block_stage.get()._new_pending_producer_schedule = producer_authority_schedule::from_shared(gpo.proposed_schedule); + std::get(pending->_block_stage)._new_pending_producer_schedule = producer_authority_schedule::from_shared(gpo.proposed_schedule); db.modify( gpo, [&]( auto& gp ) { - gp.proposed_schedule_block_num = optional(); + gp.proposed_schedule_block_num = std::optional(); gp.proposed_schedule.version=0; gp.proposed_schedule.producers.clear(); }); @@ -1643,12 +1502,13 @@ struct controller_impl { try { transaction_metadata_ptr onbtrx = - transaction_metadata::create_no_recover_keys( packed_transaction( get_on_block_transaction() ), transaction_metadata::trx_type::implicit ); + transaction_metadata::create_no_recover_keys( std::make_shared( get_on_block_transaction(), true ), + transaction_metadata::trx_type::implicit ); auto reset_in_trx_requiring_checks = fc::make_scoped_exit([old_value=in_trx_requiring_checks,this](){ in_trx_requiring_checks = old_value; }); in_trx_requiring_checks = true; - push_transaction( onbtrx, fc::time_point::maximum(), self.get_global_properties().configuration.min_transaction_cpu_usage, true, 0 ); + push_transaction( onbtrx, fc::time_point::maximum(), gpo.configuration.min_transaction_cpu_usage, true, {}, 0 ); } catch( const std::bad_alloc& e ) { elog( "on block transaction failed due to a std::bad_alloc" ); throw; @@ -1658,6 +1518,9 @@ struct controller_impl { } catch( const fc::exception& e ) { wlog( "on block transaction failed, but shouldn't impact block generation, system contract needs update" ); edump((e.to_detail_string())); + } catch( const std::exception& e ) { + wlog( "on block transaction failed due to unexpected exception" ); + edump((e.what())); } catch( ... ) { elog( "on block transaction failed due to unknown exception" ); } @@ -1672,12 +1535,27 @@ struct controller_impl { void finalize_block() { EOS_ASSERT( pending, block_validate_exception, "it is not valid to finalize when there is no pending block"); - EOS_ASSERT( pending->_block_stage.contains(), block_validate_exception, "already called finalize_block"); + EOS_ASSERT( std::holds_alternative(pending->_block_stage), block_validate_exception, "already called finalize_block"); try { auto& pbhs = pending->get_pending_block_header_state(); + auto& bb = std::get(pending->_block_stage); + + auto action_merkle_fut = async_thread_pool( thread_pool.get_executor(), + [ids{std::move( bb._action_receipt_digests )}]() mutable { + return merkle( std::move( ids ) ); + } ); + const bool calc_trx_merkle = !std::holds_alternative(bb._trx_mroot_or_receipt_digests); + std::future trx_merkle_fut; + if( calc_trx_merkle ) { + trx_merkle_fut = async_thread_pool( thread_pool.get_executor(), + [ids{std::move( std::get(bb._trx_mroot_or_receipt_digests) )}]() mutable { + return merkle( std::move( ids ) ); + } ); + } + // Update resource limits: resource_limits.process_account_limit_updates(); const auto& chain_config = self.get_global_properties().configuration; @@ -1688,12 +1566,10 @@ struct controller_impl { ); resource_limits.process_block_usage(pbhs.block_num); - auto& bb = pending->_block_stage.get(); - // Create (unsigned) block: auto block_ptr = std::make_shared( pbhs.make_block_header( - bb._transaction_mroot ? *bb._transaction_mroot : calculate_trx_merkle( bb._pending_trx_receipts ), - calculate_action_merkle(), + calc_trx_merkle ? trx_merkle_fut.get() : std::get(bb._trx_mroot_or_receipt_digests), + action_merkle_fut.get(), bb._new_pending_producer_schedule, std::move( bb._new_protocol_feature_activations ), protocol_features.get_protocol_feature_set() @@ -1701,7 +1577,7 @@ struct controller_impl { block_ptr->transactions = std::move( bb._pending_trx_receipts ); - auto id = block_ptr->id(); + auto id = block_ptr->calculate_id(); // Update TaPoS table: create_block_summary( id ); @@ -1738,10 +1614,10 @@ struct controller_impl { }); try { - EOS_ASSERT( pending->_block_stage.contains(), block_validate_exception, + EOS_ASSERT( std::holds_alternative(pending->_block_stage), block_validate_exception, "cannot call commit_block until pending block is completed" ); - auto bsp = pending->_block_stage.get()._block_state; + auto bsp = std::get(pending->_block_stage)._block_state; if( add_to_fork_db ) { fork_db.add( bsp ); @@ -1835,36 +1711,18 @@ struct controller_impl { } } - void report_block_header_diff( const block_header& b, const block_header& ab ) { - -#define EOS_REPORT(DESC,A,B) \ - if( A != B ) { \ - elog("${desc}: ${bv} != ${abv}", ("desc", DESC)("bv", A)("abv", B)); \ - } - - EOS_REPORT( "timestamp", b.timestamp, ab.timestamp ) - EOS_REPORT( "producer", b.producer, ab.producer ) - EOS_REPORT( "confirmed", b.confirmed, ab.confirmed ) - EOS_REPORT( "previous", b.previous, ab.previous ) - EOS_REPORT( "transaction_mroot", b.transaction_mroot, ab.transaction_mroot ) - EOS_REPORT( "action_mroot", b.action_mroot, ab.action_mroot ) - EOS_REPORT( "schedule_version", b.schedule_version, ab.schedule_version ) - EOS_REPORT( "new_producers", b.new_producers, ab.new_producers ) - EOS_REPORT( "header_extensions", b.header_extensions, ab.header_extensions ) - -#undef EOS_REPORT - } - - void apply_block( const block_state_ptr& bsp, controller::block_status s, const trx_meta_cache_lookup& trx_lookup ) { try { try { const signed_block_ptr& b = bsp->block; const auto& new_protocol_feature_activations = bsp->get_new_protocol_feature_activations(); - auto producer_block_id = b->id(); + auto producer_block_id = bsp->id; start_block( b->timestamp, b->confirmed, new_protocol_feature_activations, s, producer_block_id); + // validated in create_block_state_future() + std::get(pending->_block_stage)._trx_mroot_or_receipt_digests = b->transaction_mroot; + const bool existing_trxs_metas = !bsp->trxs_metas().empty(); const bool pub_keys_recovered = bsp->is_pub_keys_recovered(); const bool skip_auth_checks = self.skip_auth_check(); @@ -1875,18 +1733,19 @@ struct controller_impl { } else { trx_metas.reserve( b->transactions.size() ); for( const auto& receipt : b->transactions ) { - if( receipt.trx.contains()) { - const auto& pt = receipt.trx.get(); + if( std::holds_alternative(receipt.trx)) { + const auto& pt = std::get(receipt.trx); transaction_metadata_ptr trx_meta_ptr = trx_lookup ? trx_lookup( pt.id() ) : transaction_metadata_ptr{}; if( trx_meta_ptr && *trx_meta_ptr->packed_trx() != pt ) trx_meta_ptr = nullptr; if( trx_meta_ptr && ( skip_auth_checks || !trx_meta_ptr->recovered_keys().empty() ) ) { trx_metas.emplace_back( std::move( trx_meta_ptr ), recover_keys_future{} ); } else if( skip_auth_checks ) { + packed_transaction_ptr ptrx( b, &pt ); // alias signed_block_ptr trx_metas.emplace_back( - transaction_metadata::create_no_recover_keys( pt, transaction_metadata::trx_type::input ), + transaction_metadata::create_no_recover_keys( std::move(ptrx), transaction_metadata::trx_type::input ), recover_keys_future{} ); } else { - auto ptrx = std::make_shared( pt ); + packed_transaction_ptr ptrx( b, &pt ); // alias signed_block_ptr auto fut = transaction_metadata::start_recover_keys( std::move( ptrx ), thread_pool.get_executor(), chain_id, microseconds::maximum() ); trx_metas.emplace_back( transaction_metadata_ptr{}, std::move( fut ) ); @@ -1897,25 +1756,31 @@ struct controller_impl { transaction_trace_ptr trace; + bool explicit_net = self.skip_trx_checks(); + size_t packed_idx = 0; + const auto& trx_receipts = std::get(pending->_block_stage)._pending_trx_receipts; for( const auto& receipt : b->transactions ) { - const auto& trx_receipts = pending->_block_stage.get()._pending_trx_receipts; auto num_pending_receipts = trx_receipts.size(); - if( receipt.trx.contains() ) { + if( std::holds_alternative(receipt.trx) ) { const auto& trx_meta = ( use_bsp_cached ? bsp->trxs_metas().at( packed_idx ) : ( !!std::get<0>( trx_metas.at( packed_idx ) ) ? std::get<0>( trx_metas.at( packed_idx ) ) : std::get<1>( trx_metas.at( packed_idx ) ).get() ) ); - trace = push_transaction( trx_meta, fc::time_point::maximum(), receipt.cpu_usage_us, true, 0 ); + std::optional explicit_net_usage_words; + if( explicit_net ) { + explicit_net_usage_words = receipt.net_usage_words.value; + } + trace = push_transaction( trx_meta, fc::time_point::maximum(), receipt.cpu_usage_us, true, explicit_net_usage_words, 0 ); ++packed_idx; - } else if( receipt.trx.contains() ) { - trace = push_scheduled_transaction( receipt.trx.get(), fc::time_point::maximum(), receipt.cpu_usage_us, true ); + } else if( std::holds_alternative(receipt.trx) ) { + trace = push_scheduled_transaction( std::get(receipt.trx), fc::time_point::maximum(), receipt.cpu_usage_us, true ); } else { EOS_ASSERT( false, block_validate_exception, "encountered unexpected receipt type" ); } bool transaction_failed = trace && trace->except; - bool transaction_can_fail = receipt.status == transaction_receipt_header::hard_fail && receipt.trx.contains(); + bool transaction_can_fail = receipt.status == transaction_receipt_header::hard_fail && std::holds_alternative(receipt.trx); if( transaction_failed && !transaction_can_fail) { edump((*trace)); throw *trace->except; @@ -1923,32 +1788,25 @@ struct controller_impl { EOS_ASSERT( trx_receipts.size() > 0, block_validate_exception, "expected a receipt", - ("block", *b)("expected_receipt", receipt) + ("block_num", b->block_num())("block_id", producer_block_id)("expected_receipt", receipt) ); EOS_ASSERT( trx_receipts.size() == num_pending_receipts + 1, block_validate_exception, "expected receipt was not added", - ("block", *b)("expected_receipt", receipt) + ("block_num", b->block_num())("block_id", producer_block_id)("expected_receipt", receipt) ); const transaction_receipt_header& r = trx_receipts.back(); EOS_ASSERT( r == static_cast(receipt), block_validate_exception, "receipt does not match", - ("producer_receipt", receipt)("validator_receipt", trx_receipts.back()) ); + ("producer_receipt", static_cast(receipt))("validator_receipt", r) ); } - // validated in create_block_state_future() - pending->_block_stage.get()._transaction_mroot = b->transaction_mroot; - finalize_block(); - auto& ab = pending->_block_stage.get(); + auto& ab = std::get(pending->_block_stage); - if( producer_block_id != ab._id ) { - elog( "Validation block id does not match producer block id" ); - report_block_header_diff( *b, *ab._unsigned_block ); - // this implicitly asserts that all header fields (less the signature) are identical - EOS_ASSERT( producer_block_id == ab._id, block_validate_exception, "Block ID does not match", - ("producer_block_id", producer_block_id)("validator_block_id", ab._id) ); - } + // this implicitly asserts that all header fields (less the signature) are identical + EOS_ASSERT( producer_block_id == ab._id, block_validate_exception, "Block ID does not match", + ("producer_block_id",producer_block_id)("validator_block_id",ab._id) ); if( !use_bsp_cached ) { bsp->set_trxs_metas( std::move( ab._trx_metas ), !skip_auth_checks ); @@ -1958,18 +1816,24 @@ struct controller_impl { commit_block(false); return; + } catch ( const std::bad_alloc& ) { + throw; + } catch ( const boost::interprocess::bad_alloc& ) { + throw; } catch ( const fc::exception& e ) { edump((e.to_detail_string())); abort_block(); throw; + } catch ( const std::exception& e ) { + edump((e.what())); + abort_block(); + throw; } } FC_CAPTURE_AND_RETHROW() } /// apply_block - std::future create_block_state_future( const signed_block_ptr& b ) { + std::future create_block_state_future( const block_id_type& id, const signed_block_ptr& b ) { EOS_ASSERT( b, block_validate_exception, "null block" ); - auto id = b->id(); - // no reason for a block_state if fork_db already knows about block auto existing = fork_db.get_block( id ); EOS_ASSERT( !existing, fork_database_exception, "we already know about this block: ${id}", ("id", id) ); @@ -1978,14 +1842,14 @@ struct controller_impl { EOS_ASSERT( prev, unlinkable_block_exception, "unlinkable block ${id}", ("id", id)("previous", b->previous) ); - return async_thread_pool( thread_pool.get_executor(), [b, prev, control=this]() { + return async_thread_pool( thread_pool.get_executor(), [b, prev, id, control=this]() { const bool skip_validate_signee = false; auto trx_mroot = calculate_trx_merkle( b->transactions ); EOS_ASSERT( b->transaction_mroot == trx_mroot, block_validate_exception, "invalid block transaction merkle root ${b} != ${c}", ("b", b->transaction_mroot)("c", trx_mroot) ); - return std::make_shared( + auto bsp = std::make_shared( *prev, move( b ), control->protocol_features.get_protocol_feature_set(), @@ -1995,10 +1859,14 @@ struct controller_impl { { control->check_protocol_features( timestamp, cur_features, new_features ); }, skip_validate_signee ); + + EOS_ASSERT( id == bsp->id, block_validate_exception, + "provided id ${id} does not match block id ${bid}", ("id", id)("bid", bsp->id) ); + return bsp; } ); } - void push_block( std::future& block_state_future, + block_state_ptr push_block( std::future& block_state_future, const forked_branch_callback& forked_branch_cb, const trx_meta_cache_lookup& trx_lookup ) { controller::block_status s = controller::block_status::complete; @@ -2011,6 +1879,12 @@ struct controller_impl { block_state_ptr bsp = block_state_future.get(); const auto& b = bsp->block; + if( conf.terminate_at_block > 0 && conf.terminate_at_block < b->block_num() ) { + ilog("Reached configured maximum block ${num}; terminating", ("num", conf.terminate_at_block) ); + shutdown(); + return bsp; + } + emit( self.pre_accepted_block, b ); fork_db.add( bsp ); @@ -2026,8 +1900,9 @@ struct controller_impl { } else { log_irreversible(); } - - } FC_LOG_AND_RETHROW( ) + return bsp; + } + FC_LOG_AND_RETHROW() } void replay_push_block( const signed_block_ptr& b, controller::block_status s ) { @@ -2040,6 +1915,13 @@ struct controller_impl { EOS_ASSERT( b, block_validate_exception, "trying to push empty block" ); EOS_ASSERT( (s == controller::block_status::irreversible || s == controller::block_status::validated), block_validate_exception, "invalid block status for replay" ); + + if( conf.terminate_at_block > 0 && conf.terminate_at_block < b->block_num() ) { + ilog("Reached configured maximum block ${num}; terminating", ("num", conf.terminate_at_block) ); + shutdown(); + return; + } + emit( self.pre_accepted_block, b ); const bool skip_validate_signee = !conf.force_all_checks; @@ -2064,12 +1946,12 @@ struct controller_impl { apply_block( bsp, s, trx_meta_cache_lookup{} ); head = bsp; - // On replay, log_irreversible is not called and so no irreversible_block signal is emittted. + // On replay, log_irreversible is not called and so no irreversible_block signal is emitted. // So emit it explicitly here. emit( self.irreversible_block, bsp ); if (!self.skip_db_sessions(s)) { - db.commit(bsp->block_num); + kv_db.commit(bsp->block_num); } } else { @@ -2090,7 +1972,7 @@ struct controller_impl { apply_block( new_head, s, trx_lookup ); fork_db.mark_valid( new_head ); head = new_head; - } catch ( const fc::exception& e ) { + } catch ( const std::exception& e ) { fork_db.remove( new_head->id ); throw; } @@ -2098,6 +1980,14 @@ struct controller_impl { auto old_head = head; ilog("switching forks from ${current_head_id} (block number ${current_head_num}) to ${new_head_id} (block number ${new_head_num})", ("current_head_id", head->id)("current_head_num", head->block_num)("new_head_id", new_head->id)("new_head_num", new_head->block_num) ); + + if (auto dm_logger = get_deep_mind_logger()) { + fc_dlog(*dm_logger, "SWITCH_FORK ${from_id} ${to_id}", + ("from_id", head->id) + ("to_id", new_head->id) + ); + } + auto branches = fork_db.fetch_branch_from( new_head->id, head->id ); if( branches.second.size() > 0 ) { @@ -2111,18 +2001,25 @@ struct controller_impl { } for( auto ritr = branches.first.rbegin(); ritr != branches.first.rend(); ++ritr ) { - optional except; + auto except = std::exception_ptr{}; try { apply_block( *ritr, (*ritr)->is_valid() ? controller::block_status::validated : controller::block_status::complete, trx_lookup ); fork_db.mark_valid( *ritr ); head = *ritr; + } catch ( const std::bad_alloc& ) { + throw; + } catch ( const boost::interprocess::bad_alloc& ) { + throw; } catch (const fc::exception& e) { - except = e; + elog("exception thrown while switching forks ${e}", ("e", e.to_detail_string())); + except = std::current_exception(); + } catch (const std::exception& e) { + elog("exception thrown while switching forks ${e}", ("e", e.what())); + except = std::current_exception(); } - if( except ) { - elog("exception thrown while switching forks ${e}", ("e", except->to_detail_string())); + if( except ) { // ritr currently points to the block that threw // Remove the block that threw and all forks built off it. fork_db.remove( (*ritr)->id ); @@ -2141,7 +2038,7 @@ struct controller_impl { apply_block( *ritr, controller::block_status::validated /* we previously validated these blocks*/, trx_lookup ); head = *ritr; } - throw *except; + std::rethrow_exception(except); } // end if exception } /// end for each block in branch @@ -2155,8 +2052,8 @@ struct controller_impl { } /// push_block - vector abort_block() { - vector applied_trxs; + deque abort_block() { + deque applied_trxs; if( pending ) { applied_trxs = pending->extract_trx_metas(); pending.reset(); @@ -2165,23 +2062,11 @@ struct controller_impl { return applied_trxs; } - checksum256_type calculate_action_merkle() { - vector action_digests; - const auto& actions = pending->_block_stage.get()._actions; - action_digests.reserve( actions.size() ); - for( const auto& a : actions ) - action_digests.emplace_back( a.digest() ); - - return merkle( move(action_digests) ); - } - - static checksum256_type calculate_trx_merkle( const vector& trxs ) { - vector trx_digests; - trx_digests.reserve( trxs.size() ); + static checksum256_type calculate_trx_merkle( const deque& trxs ) { + deque trx_digests; for( const auto& a : trxs ) trx_digests.emplace_back( a.digest() ); - - return merkle( move(trx_digests) ); + return merkle( move( trx_digests ) ); } void update_producers_authority() { @@ -2392,7 +2277,7 @@ struct controller_impl { { action on_block_act; on_block_act.account = config::system_account_name; - on_block_act.name = N(onblock); + on_block_act.name = "onblock"_n; on_block_act.authorization = vector{{config::system_account_name, config::active_name}}; on_block_act.data = fc::raw::pack(self.head_block_header()); @@ -2406,9 +2291,23 @@ struct controller_impl { trx.expiration = self.pending_block_time() + fc::microseconds(999'999); // Round up to nearest second to avoid appearing expired trx.set_reference_block( self.head_block_id() ); } + + if (auto dm_logger = get_deep_mind_logger()) { + auto packed_trx = fc::raw::pack(trx); + + fc_dlog(*dm_logger, "TRX_OP CREATE onblock ${id} ${trx}", + ("id", trx.id()) + ("trx", fc::to_hex(packed_trx)) + ); + } + return trx; } + inline fc::logger* get_deep_mind_logger() const { + return deep_mind_logger; + } + }; /// controller_impl const resource_limits_manager& controller::get_resource_limits_manager()const @@ -2439,6 +2338,11 @@ uint32_t controller::get_max_nonprivileged_inline_action_size()const return my->conf.max_nonprivileged_inline_action_size; } +const controller::config& controller::get_config()const +{ + return my->conf; +} + controller::controller( const controller::config& cfg, const chain_id_type& chain_id ) :my( new controller_impl( cfg, *this, protocol_feature_set{}, chain_id ) ) { @@ -2457,22 +2361,24 @@ controller::~controller() { //for that we need 'my' to be valid pointer pointing to valid controller_impl. my->fork_db.close(); */ + + my->kv_db.flush(); } void controller::add_indices() { my->add_indices(); } -void controller::startup( std::function shutdown, const snapshot_reader_ptr& snapshot ) { - my->startup(shutdown, snapshot); +void controller::startup( std::function shutdown, std::function check_shutdown, const snapshot_reader_ptr& snapshot ) { + my->startup(shutdown, check_shutdown, snapshot); } -void controller::startup( std::function shutdown, const genesis_state& genesis ) { - my->startup(shutdown, genesis); +void controller::startup( std::function shutdown, std::function check_shutdown, const genesis_state& genesis ) { + my->startup(shutdown, check_shutdown, genesis); } -void controller::startup( std::function shutdown ) { - my->startup(shutdown); +void controller::startup(std::function shutdown, std::function check_shutdown) { + my->startup(shutdown, check_shutdown); } const chainbase::database& controller::db()const { return my->db; } @@ -2480,8 +2386,11 @@ const chainbase::database& controller::db()const { return my->db; } chainbase::database& controller::mutable_db()const { return my->db; } const fork_database& controller::fork_db()const { return my->fork_db; } +eosio::chain::combined_database& controller::kv_db() const { return my->kv_db; } -void controller::preactivate_feature( const digest_type& feature_digest ) { +const chainbase::database& controller::reversible_db()const { return my->reversible_blocks; } + +void controller::preactivate_feature( uint32_t action_id, const digest_type& feature_digest ) { const auto& pfs = my->protocol_features.get_protocol_feature_set(); auto cur_time = pending_block_time(); @@ -2578,6 +2487,16 @@ void controller::preactivate_feature( const digest_type& feature_digest ) { ("digest", feature_digest) ); + if (auto dm_logger = get_deep_mind_logger()) { + const auto feature = pfs.get_protocol_feature(feature_digest); + + fc_dlog(*dm_logger, "FEATURE_OP PRE_ACTIVATE ${action_id} ${feature_digest} ${feature}", + ("action_id", action_id) + ("feature_digest", feature_digest) + ("feature", feature.to_variant()) + ); + } + my->db.modify( pso, [&]( auto& ps ) { ps.preactivated_protocol_features.push_back( feature_digest ); } ); @@ -2623,7 +2542,7 @@ void controller::start_block( block_timestamp_type when, uint16_t confirm_block_ } my->start_block( when, confirm_block_count, new_protocol_feature_activations, - block_status::incomplete, optional() ); + block_status::incomplete, std::optional() ); } void controller::start_block( block_timestamp_type when, @@ -2637,7 +2556,7 @@ void controller::start_block( block_timestamp_type when, } my->start_block( when, confirm_block_count, new_protocol_feature_activations, - block_status::incomplete, optional() ); + block_status::incomplete, std::optional() ); } block_state_ptr controller::finalize_block( const signer_callback_type& signer_callback ) { @@ -2645,7 +2564,7 @@ block_state_ptr controller::finalize_block( const signer_callback_type& signer_c my->finalize_block(); - auto& ab = my->pending->_block_stage.get(); + auto& ab = std::get(my->pending->_block_stage); auto bsp = std::make_shared( std::move( ab._pending_block_header_state ), @@ -2670,7 +2589,7 @@ void controller::commit_block() { my->commit_block(true); } -vector controller::abort_block() { +deque controller::abort_block() { return my->abort_block(); } @@ -2678,16 +2597,16 @@ boost::asio::io_context& controller::get_thread_pool() { return my->thread_pool.get_executor(); } -std::future controller::create_block_state_future( const signed_block_ptr& b ) { - return my->create_block_state_future( b ); +std::future controller::create_block_state_future( const block_id_type& id, const signed_block_ptr& b ) { + return my->create_block_state_future( id, b ); } -void controller::push_block( std::future& block_state_future, +block_state_ptr controller::push_block( std::future& block_state_future, const forked_branch_callback& forked_branch_cb, const trx_meta_cache_lookup& trx_lookup ) { validate_db_available_size(); validate_reversible_available_size(); - my->push_block( block_state_future, forked_branch_cb, trx_lookup ); + return my->push_block( block_state_future, forked_branch_cb, trx_lookup ); } transaction_trace_ptr controller::push_transaction( const transaction_metadata_ptr& trx, fc::time_point deadline, @@ -2696,7 +2615,7 @@ transaction_trace_ptr controller::push_transaction( const transaction_metadata_p validate_db_available_size(); EOS_ASSERT( get_read_mode() != db_read_mode::IRREVERSIBLE, transaction_type_exception, "push transaction not allowed in irreversible mode" ); EOS_ASSERT( trx && !trx->implicit && !trx->scheduled, transaction_type_exception, "Implicit/Scheduled transaction not allowed" ); - return my->push_transaction(trx, deadline, billed_cpu_time_us, explicit_billed_cpu_time, subjective_cpu_bill_us ); + return my->push_transaction(trx, deadline, billed_cpu_time_us, explicit_billed_cpu_time, {}, subjective_cpu_bill_us ); } transaction_trace_ptr controller::push_scheduled_transaction( const transaction_id_type& trxid, fc::time_point deadline, @@ -2803,8 +2722,8 @@ account_name controller::fork_db_pending_head_block_producer()const { time_point controller::pending_block_time()const { EOS_ASSERT( my->pending, block_validate_exception, "no pending block" ); - if( my->pending->_block_stage.contains() ) - return my->pending->_block_stage.get()._block_state->header.timestamp; + if( std::holds_alternative(my->pending->_block_stage) ) + return std::get(my->pending->_block_stage)._block_state->header.timestamp; return my->pending->get_pending_block_header_state().timestamp; } @@ -2812,8 +2731,8 @@ time_point controller::pending_block_time()const { account_name controller::pending_block_producer()const { EOS_ASSERT( my->pending, block_validate_exception, "no pending block" ); - if( my->pending->_block_stage.contains() ) - return my->pending->_block_stage.get()._block_state->header.producer; + if( std::holds_alternative(my->pending->_block_stage) ) + return std::get(my->pending->_block_stage)._block_state->header.producer; return my->pending->get_pending_block_header_state().producer; } @@ -2821,18 +2740,18 @@ account_name controller::pending_block_producer()const { const block_signing_authority& controller::pending_block_signing_authority()const { EOS_ASSERT( my->pending, block_validate_exception, "no pending block" ); - if( my->pending->_block_stage.contains() ) - return my->pending->_block_stage.get()._block_state->valid_block_signing_authority; + if( std::holds_alternative(my->pending->_block_stage) ) + return std::get(my->pending->_block_stage)._block_state->valid_block_signing_authority; return my->pending->get_pending_block_header_state().valid_block_signing_authority; } -optional controller::pending_producer_block_id()const { +std::optional controller::pending_producer_block_id()const { EOS_ASSERT( my->pending, block_validate_exception, "no pending block" ); return my->pending->_producer_block_id; } -const vector& controller::get_pending_trx_receipts()const { +const deque& controller::get_pending_trx_receipts()const { EOS_ASSERT( my->pending, block_validate_exception, "no pending block" ); return my->pending->get_trx_receipts(); } @@ -2851,6 +2770,9 @@ time_point controller::last_irreversible_block_time() const { return my->fork_db.root()->header.timestamp.to_time_point(); } +const signed_block_ptr controller::last_irreversible_block() const { + return my->blog.head(); +} const dynamic_global_property_object& controller::get_dynamic_global_properties()const { return my->db.get(); @@ -2863,7 +2785,7 @@ signed_block_ptr controller::fetch_block_by_id( block_id_type id )const { auto state = my->fork_db.get_block(id); if( state && state->block ) return state->block; auto bptr = fetch_block_by_number( block_header::num_from_id(id) ); - if( bptr && bptr->id() == id ) return bptr; + if( bptr && bptr->calculate_id() == id ) return bptr; return signed_block_ptr(); } @@ -2873,7 +2795,7 @@ signed_block_ptr controller::fetch_block_by_number( uint32_t block_num )const { return blk_state->block; } - return my->blog.read_block_by_num(block_num); + return my->blog.read_signed_block_by_num(block_num); } FC_CAPTURE_AND_RETHROW( (block_num) ) } block_state_ptr controller::fetch_block_state_by_id( block_id_type id )const { @@ -2934,7 +2856,7 @@ sha256 controller::calculate_integrity_hash()const { try { void controller::write_snapshot( const snapshot_writer_ptr& snapshot ) const { EOS_ASSERT( !my->pending, block_validate_exception, "cannot take a consistent snapshot with a pending block" ); - return my->add_to_snapshot(snapshot); + return my->kv_db.add_to_snapshot(snapshot, *my->fork_db.head(), my->authorization, my->resource_limits); } int64_t controller::set_proposed_producers( vector producers ) { @@ -2945,7 +2867,7 @@ int64_t controller::set_proposed_producers( vector producers return -1; } - if( gpo.proposed_schedule_block_num.valid() ) { + if( gpo.proposed_schedule_block_num ) { if( *gpo.proposed_schedule_block_num != cur_block_num ) return -1; // there is already a proposed schedule set in a previous block, wait for it to become pending @@ -2992,8 +2914,8 @@ const producer_authority_schedule& controller::active_producers()const { if( !(my->pending) ) return my->head->active_schedule; - if( my->pending->_block_stage.contains() ) - return my->pending->_block_stage.get()._block_state->active_schedule; + if( std::holds_alternative(my->pending->_block_stage) ) + return std::get(my->pending->_block_stage)._block_state->active_schedule; return my->pending->get_pending_block_header_state().active_schedule; } @@ -3002,17 +2924,17 @@ const producer_authority_schedule& controller::pending_producers()const { if( !(my->pending) ) return my->head->pending_schedule.schedule; - if( my->pending->_block_stage.contains() ) - return my->pending->_block_stage.get()._block_state->pending_schedule.schedule; + if( std::holds_alternative(my->pending->_block_stage) ) + return std::get(my->pending->_block_stage)._block_state->pending_schedule.schedule; - if( my->pending->_block_stage.contains() ) { - const auto& new_prods_cache = my->pending->_block_stage.get()._new_producer_authority_cache; + if( std::holds_alternative(my->pending->_block_stage) ) { + const auto& new_prods_cache = std::get(my->pending->_block_stage)._new_producer_authority_cache; if( new_prods_cache ) { return *new_prods_cache; } } - const auto& bb = my->pending->_block_stage.get(); + const auto& bb = std::get(my->pending->_block_stage); if( bb._new_pending_producer_schedule ) return *bb._new_pending_producer_schedule; @@ -3020,15 +2942,15 @@ const producer_authority_schedule& controller::pending_producers()const { return bb._pending_block_header_state.prev_pending_schedule.schedule; } -optional controller::proposed_producers()const { +std::optional controller::proposed_producers()const { const auto& gpo = get_global_properties(); - if( !gpo.proposed_schedule_block_num.valid() ) - return optional(); + if( !gpo.proposed_schedule_block_num ) + return std::optional(); return producer_authority_schedule::from_shared(gpo.proposed_schedule); } -bool controller::light_validation_allowed(bool replay_opts_disabled_by_policy) const { +bool controller::light_validation_allowed() const { if (!my->pending || my->in_trx_requiring_checks) { return false; } @@ -3036,7 +2958,8 @@ bool controller::light_validation_allowed(bool replay_opts_disabled_by_policy) c const auto pb_status = my->pending->_block_status; // in a pending irreversible or previously validated block and we have forcing all checks - const bool consider_skipping_on_replay = (pb_status == block_status::irreversible || pb_status == block_status::validated) && !replay_opts_disabled_by_policy; + const bool consider_skipping_on_replay = + (pb_status == block_status::irreversible || pb_status == block_status::validated) && !my->conf.force_all_checks; // OR in a signed block and in light validation mode const bool consider_skipping_on_validate = (pb_status == block_status::complete && @@ -3047,7 +2970,11 @@ bool controller::light_validation_allowed(bool replay_opts_disabled_by_policy) c bool controller::skip_auth_check() const { - return light_validation_allowed(my->conf.force_all_checks); + return light_validation_allowed(); +} + +bool controller::skip_trx_checks() const { + return light_validation_allowed(); } bool controller::skip_db_sessions( block_status bs ) const { @@ -3057,7 +2984,7 @@ bool controller::skip_db_sessions( block_status bs ) const { && !my->in_trx_requiring_checks; } -bool controller::skip_db_sessions( ) const { +bool controller::skip_db_sessions() const { if (my->pending) { return skip_db_sessions(my->pending->_block_status); } else { @@ -3065,10 +2992,6 @@ bool controller::skip_db_sessions( ) const { } } -bool controller::skip_trx_checks() const { - return light_validation_allowed(my->conf.disable_replay_opts); -} - bool controller::is_trusted_producer( const account_name& producer) const { return get_validation_mode() == chain::validation_mode::LIGHT || my->conf.trusted_producers.count(producer); } @@ -3089,6 +3012,14 @@ validation_mode controller::get_validation_mode()const { return my->conf.block_validation_mode; } +const flat_set& controller::get_trusted_producers()const { + return my->conf.trusted_producers; +} + +uint32_t controller::get_terminate_at_block()const { + return my->conf.terminate_at_block; +} + const apply_handler* controller::find_apply_handler( account_name receiver, account_name scope, action_name act ) const { auto native_handler_scope = my->apply_handlers.find( receiver ); @@ -3129,7 +3060,7 @@ void controller::check_key_list( const public_key_type& key )const { } bool controller::is_building_block()const { - return my->pending.valid(); + return my->pending.has_value(); } bool controller::is_producing_block()const { @@ -3209,7 +3140,7 @@ void controller::set_subjective_cpu_leeway(fc::microseconds leeway) { my->subjective_cpu_leeway = leeway; } -fc::optional controller::get_subjective_cpu_leeway() const { +std::optional controller::get_subjective_cpu_leeway() const { return my->subjective_cpu_leeway; } @@ -3244,30 +3175,57 @@ const flat_set &controller::get_resource_greylist() const { } -void controller::add_to_ram_correction( account_name account, uint64_t ram_bytes ) { +void controller::add_to_ram_correction( account_name account, uint64_t ram_bytes, uint32_t action_id, const char* event_id ) { + int64_t correction_object_id = 0; + if( auto ptr = my->db.find( account ) ) { my->db.modify( *ptr, [&]( auto& rco ) { + correction_object_id = rco.id._id; rco.ram_correction += ram_bytes; } ); } else { my->db.create( [&]( auto& rco ) { + correction_object_id = rco.id._id; rco.name = account; rco.ram_correction = ram_bytes; } ); } + + if (auto dm_logger = get_deep_mind_logger()) { + fc_dlog(*dm_logger, "RAM_CORRECTION_OP ${action_id} ${correction_id} ${event_id} ${payer} ${delta}", + ("action_id", action_id) + ("correction_id", correction_object_id) + ("event_id", event_id) + ("payer", account) + ("delta", ram_bytes) + ); + } +} + +fc::microseconds controller::get_abi_serializer_max_time()const { + return my->conf.abi_serializer_max_time_us; } bool controller::all_subjective_mitigations_disabled()const { return my->conf.disable_all_subjective_mitigations; } +fc::logger* controller::get_deep_mind_logger()const { + return my->get_deep_mind_logger(); +} + +void controller::enable_deep_mind(fc::logger* logger) { + EOS_ASSERT( logger != nullptr, misc_exception, "Invalid logger passed into enable_deep_mind, must be set" ); + my->deep_mind_logger = logger; +} + #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED) vm::wasm_allocator& controller::get_wasm_allocator() { return my->wasm_alloc; } #endif -fc::optional controller::convert_exception_to_error_code( const fc::exception& e ) { +std::optional controller::convert_exception_to_error_code( const fc::exception& e ) { const chain_exception* e_ptr = dynamic_cast( &e ); if( e_ptr == nullptr ) return {}; @@ -3285,21 +3243,32 @@ chain_id_type controller::extract_chain_id(snapshot_reader& snapshot) { }); // check if this is a legacy version of the snapshot, which has a genesis state instead of chain id - fc::optional genesis = controller_impl::extract_legacy_genesis_state(snapshot, header.version); + std::optional genesis = extract_legacy_genesis_state(snapshot, header.version); if (genesis) { return genesis->compute_chain_id(); } chain_id_type chain_id; - snapshot.read_section([&chain_id]( auto §ion ){ - snapshot_global_property_object global_properties; - section.read_row(global_properties); - chain_id = global_properties.chain_id; - }); + using v4 = legacy::snapshot_global_property_object_v4; + if (header.version <= v4::maximum_version) { + snapshot.read_section([&chain_id]( auto §ion ){ + v4 global_properties; + section.read_row(global_properties); + chain_id = global_properties.chain_id; + }); + } + else { + snapshot.read_section([&chain_id]( auto §ion ){ + snapshot_global_property_object global_properties; + section.read_row(global_properties); + chain_id = global_properties.chain_id; + }); + } + return chain_id; } -fc::optional controller::extract_chain_id_from_db( const path& state_dir ) { +std::optional controller::extract_chain_id_from_db( const path& state_dir ) { try { chainbase::database db( state_dir, chainbase::database::read_only ); @@ -3319,6 +3288,39 @@ fc::optional controller::extract_chain_id_from_db( const path& st return {}; } +void controller::replace_producer_keys( const public_key_type& key ) { + ilog("Replace producer keys with ${k}", ("k", key)); + mutable_db().modify( db().get(), [&]( auto& gp ) { + gp.proposed_schedule_block_num = {}; + gp.proposed_schedule.version = 0; + gp.proposed_schedule.producers.clear(); + }); + auto version = my->head->pending_schedule.schedule.version; + my->head->pending_schedule = {}; + my->head->pending_schedule.schedule.version = version; + for (auto& prod: my->head->active_schedule.producers ) { + ilog("${n}", ("n", prod.producer_name)); + std::visit([&](auto &auth) { + auth.threshold = 1; + auth.keys = {key_weight{key, 1}}; + }, prod.authority); + } +} + +void controller::replace_account_keys( name account, name permission, const public_key_type& key ) { + auto& rlm = get_mutable_resource_limits_manager(); + auto* perm = db().find(boost::make_tuple(account, permission)); + if (!perm) + return; + int64_t old_size = (int64_t)(chain::config::billable_size_v + perm->auth.get_billable_size()); + mutable_db().modify(*perm, [&](auto& p) { + p.auth = authority(key); + }); + int64_t new_size = (int64_t)(chain::config::billable_size_v + perm->auth.get_billable_size()); + rlm.add_pending_ram_usage(account, new_size - old_size, generic_storage_usage_trace(0)); + rlm.verify_account_ram_usage(account); +} + /// Protocol feature activation handlers: template<> @@ -3348,7 +3350,12 @@ void controller_impl::on_activationname)("adjust", itr->ram_correction)("current", current_ram_usage) ); } - resource_limits.add_pending_ram_usage( itr->name, ram_delta ); + std::string event_id; + if (get_deep_mind_logger() != nullptr) { + event_id = STORAGE_EVENT_ID("${id}", ("id", itr->id._id)); + } + + resource_limits.add_pending_ram_usage( itr->name, ram_delta, storage_usage_trace(0, std::move(event_id), "deferred_trx", "correction", "deferred_trx_ram_correction") ); db.remove( *itr ); } } @@ -3367,7 +3374,54 @@ void controller_impl::on_activation +void controller_impl::on_activation() { + db.modify( db.get(), [&]( auto& ps ) { + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "set_action_return_value" ); + } ); +} +template<> +void controller_impl::on_activation() { + db.modify( db.get(), [&]( auto& ps ) { + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "kv_erase" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "kv_set" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "kv_get" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "kv_get_data" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "kv_it_create" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "kv_it_destroy" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "kv_it_status" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "kv_it_compare" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "kv_it_key_compare" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "kv_it_move_to_end" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "kv_it_next" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "kv_it_prev" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "kv_it_lower_bound" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "kv_it_key" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "kv_it_value" ); + // resource management + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "set_resource_limit" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "get_resource_limit" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "set_kv_parameters_packed" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "get_kv_parameters_packed" ); + } ); +} + +template<> +void controller_impl::on_activation() { + db.modify( db.get(), [&]( auto& ps ) { + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "set_wasm_parameters_packed" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "get_wasm_parameters_packed" ); + } ); +} + +template<> +void controller_impl::on_activation() { + db.modify( db.get(), [&]( auto& ps ) { + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "get_parameters_packed" ); + add_intrinsic_to_whitelist( ps.whitelisted_intrinsics, "set_parameters_packed" ); + } ); +} /// End of protocol feature activation handlers diff --git a/libraries/chain/eosio_contract.cpp b/libraries/chain/eosio_contract.cpp index ccb85a82ac2..edf8821e0d8 100644 --- a/libraries/chain/eosio_contract.cpp +++ b/libraries/chain/eosio_contract.cpp @@ -25,7 +25,7 @@ namespace eosio { namespace chain { uint128_t transaction_id_to_sender_id( const transaction_id_type& tid ) { - fc::uint128_t _id(tid._hash[3], tid._hash[2]); + fc::uint128 _id(tid._hash[3], tid._hash[2]); return (unsigned __int128)_id; } @@ -106,9 +106,9 @@ void apply_eosio_newaccount(apply_context& context) { } const auto& owner_permission = authorization.create_permission( create.name, config::owner_name, 0, - std::move(create.owner) ); + std::move(create.owner), context.get_action_id() ); const auto& active_permission = authorization.create_permission( create.name, config::active_name, owner_permission.id, - std::move(create.active) ); + std::move(create.active), context.get_action_id() ); context.control.get_mutable_resource_limits_manager().initialize_account(create.name); @@ -117,7 +117,12 @@ void apply_eosio_newaccount(apply_context& context) { ram_delta += owner_permission.auth.get_billable_size(); ram_delta += active_permission.auth.get_billable_size(); - context.add_ram_usage(create.name, ram_delta); + std::string event_id; + if (context.control.get_deep_mind_logger() != nullptr) { + event_id = STORAGE_EVENT_ID("${name}", ("name", create.name)); + } + + context.add_ram_usage(create.name, ram_delta, storage_usage_trace(context.get_action_id(), std::move(event_id), "account", "add", "newaccount")); } FC_CAPTURE_AND_RETHROW( (create) ) } @@ -191,7 +196,20 @@ void apply_eosio_setcode(apply_context& context) { }); if (new_size != old_size) { - context.add_ram_usage( act.account, new_size - old_size ); + const char* operation = ""; + std::string event_id; + if (context.control.get_deep_mind_logger() != nullptr) { + operation = "update"; + if (old_size <= 0) { + operation = "add"; + } else if (new_size <= 0) { + operation = "remove"; + } + + event_id = STORAGE_EVENT_ID("${account}", ("account", act.account)); + } + + context.add_ram_usage( act.account, new_size - old_size, storage_usage_trace(context.get_action_id(), std::move(event_id), "code", operation, "setcode") ); } } @@ -209,11 +227,7 @@ void apply_eosio_setabi(apply_context& context) { int64_t new_size = abi_size; db.modify( account, [&]( auto& a ) { - if (abi_size > 0) { - a.abi.assign(act.abi.data(), abi_size); - } else { - a.abi.resize(0); - } + a.abi.assign(act.abi.data(), abi_size); }); const auto& account_metadata = db.get(act.account); @@ -222,7 +236,20 @@ void apply_eosio_setabi(apply_context& context) { }); if (new_size != old_size) { - context.add_ram_usage( act.account, new_size - old_size ); + const char* operation = ""; + std::string event_id; + if (context.control.get_deep_mind_logger() != nullptr) { + operation = "update"; + if (old_size <= 0) { + operation = "add"; + } else if (new_size <= 0) { + operation = "remove"; + } + + event_id = STORAGE_EVENT_ID("${account}", ("account", act.account)); + } + + context.add_ram_usage( act.account, new_size - old_size, storage_usage_trace(context.get_action_id(), std::move(event_id), "abi", operation, "setabi") ); } } @@ -276,17 +303,27 @@ void apply_eosio_updateauth(apply_context& context) { int64_t old_size = (int64_t)(config::billable_size_v + permission->auth.get_billable_size()); - authorization.modify_permission( *permission, update.auth ); + authorization.modify_permission( *permission, update.auth, context.get_action_id() ); int64_t new_size = (int64_t)(config::billable_size_v + permission->auth.get_billable_size()); - context.add_ram_usage( permission->owner, new_size - old_size ); + std::string event_id; + if (context.control.get_deep_mind_logger() != nullptr) { + event_id = STORAGE_EVENT_ID("${id}", ("id", permission->id)); + } + + context.add_ram_usage( permission->owner, new_size - old_size, storage_usage_trace(context.get_action_id(), std::move(event_id), "auth", "update", "updateauth_update") ); } else { - const auto& p = authorization.create_permission( update.account, update.permission, parent_id, update.auth ); + const auto& p = authorization.create_permission( update.account, update.permission, parent_id, update.auth, context.get_action_id() ); int64_t new_size = (int64_t)(config::billable_size_v + p.auth.get_billable_size()); - context.add_ram_usage( update.account, new_size ); + std::string event_id; + if (context.control.get_deep_mind_logger() != nullptr) { + event_id = STORAGE_EVENT_ID("${id}", ("id", p.id)); + } + + context.add_ram_usage( update.account, new_size, storage_usage_trace(context.get_action_id(), std::move(event_id), "auth", "add", "updateauth_create") ); } } @@ -315,10 +352,14 @@ void apply_eosio_deleteauth(apply_context& context) { const auto& permission = authorization.get_permission({remove.account, remove.permission}); int64_t old_size = config::billable_size_v + permission.auth.get_billable_size(); - authorization.remove_permission( permission ); + std::string event_id; + if (context.control.get_deep_mind_logger() != nullptr) { + event_id = STORAGE_EVENT_ID("${id}", ("id", permission.id)); + } - context.add_ram_usage( remove.account, -old_size ); + authorization.remove_permission( permission, context.get_action_id() ); + context.add_ram_usage( remove.account, -old_size, storage_usage_trace(context.get_action_id(), std::move(event_id), "auth", "remove", "deleteauth") ); } void apply_eosio_linkauth(apply_context& context) { @@ -368,9 +409,15 @@ void apply_eosio_linkauth(apply_context& context) { link.required_permission = requirement.requirement; }); + std::string event_id; + if (context.control.get_deep_mind_logger() != nullptr) { + event_id = STORAGE_EVENT_ID("${id}", ("id", l.id)); + } + context.add_ram_usage( l.account, - (int64_t)(config::billable_size_v) + (int64_t)(config::billable_size_v), + storage_usage_trace(context.get_action_id(), std::move(event_id), "auth_link", "add", "linkauth") ); } @@ -388,9 +435,16 @@ void apply_eosio_unlinkauth(apply_context& context) { auto link_key = boost::make_tuple(unlink.account, unlink.code, unlink.type); auto link = db.find(link_key); EOS_ASSERT(link != nullptr, action_validate_exception, "Attempting to unlink authority, but no link found"); + + std::string event_id; + if (context.control.get_deep_mind_logger() != nullptr) { + event_id = STORAGE_EVENT_ID("${id}", ("id", link->id)); + } + context.add_ram_usage( link->account, - -(int64_t)(config::billable_size_v) + -(int64_t)(config::billable_size_v), + storage_usage_trace(context.get_action_id(), std::move(event_id), "auth_link", "remove", "unlinkauth") ); db.remove(*link); diff --git a/libraries/chain/fork_database.cpp b/libraries/chain/fork_database.cpp index 5fe1c05118e..379b4385929 100644 --- a/libraries/chain/fork_database.cpp +++ b/libraries/chain/fork_database.cpp @@ -277,7 +277,7 @@ namespace eosio { namespace chain { // Even though fork database no longer needs block or trxs when a block state becomes a root of the tree, // avoid mutating the block state at all, for example clearing the block shared pointer, because other - // parts of the code which run asynchronously (e.g. mongo_db_plugin) may later expect it remain unmodified. + // parts of the code which run asynchronously may later expect it remain unmodified. my->root = new_root; } @@ -315,7 +315,7 @@ namespace eosio { namespace chain { const auto& exts = n->header_exts; if( exts.count(protocol_feature_activation::extension_id()) > 0 ) { - const auto& new_protocol_features = exts.lower_bound(protocol_feature_activation::extension_id())->second.get().protocol_features; + const auto& new_protocol_features = std::get(exts.lower_bound(protocol_feature_activation::extension_id())->second).protocol_features; validator( n->header.timestamp, prev_bh->activated_protocol_features->protocol_features, new_protocol_features ); } } EOS_RETHROW_EXCEPTIONS( fork_database_exception, "serialized fork database is incompatible with configured protocol features" ) diff --git a/libraries/chain/genesis_state.cpp b/libraries/chain/genesis_state.cpp index 3a6e1650b70..03c8323f0cb 100644 --- a/libraries/chain/genesis_state.cpp +++ b/libraries/chain/genesis_state.cpp @@ -1,8 +1,5 @@ #include -// these are required to serialize a genesis_state -#include // required for gcc in release mode - namespace eosio { namespace chain { genesis_state::genesis_state() { diff --git a/libraries/chain/include/eosio/chain/abi_def.hpp b/libraries/chain/include/eosio/chain/abi_def.hpp index dcace005520..18fee7a20d0 100644 --- a/libraries/chain/include/eosio/chain/abi_def.hpp +++ b/libraries/chain/include/eosio/chain/abi_def.hpp @@ -6,6 +6,7 @@ namespace eosio { namespace chain { using type_name = string; using field_name = string; +using index_name = name; struct type_def { type_def() = default; @@ -70,6 +71,61 @@ struct table_def { type_name type; // type of binary data stored in this table }; +struct primary_index_def { + primary_index_def() = default; + primary_index_def(const index_name &name, const type_name &type) + : name(name), type(type) + {} + + index_name name; // the name of primary index + type_name type; // the kind of index + + bool operator==(const primary_index_def &other) const { + return std::tie(name, type) == std::tie(other.name, other.type); + } + + bool operator!=(const primary_index_def &other) const { + return !operator==(other); + } +}; + +struct secondary_index_def +{ + secondary_index_def() = default; + secondary_index_def(const type_name &type) + : type(type) + {} + + type_name type; // the kind of index + + bool operator==(const secondary_index_def &other) const { + return type == other.type; + } + + bool operator!=(const secondary_index_def &other) const { + return !operator==(other); + } +}; + +struct kv_table_def { + kv_table_def() = default; + kv_table_def(const type_name& type, const primary_index_def& primary_index, const map& secondary_indices) + :type(type), primary_index(primary_index), secondary_indices(secondary_indices) + {} + + type_name type; // the name of the struct_def + primary_index_def primary_index; // primary index field + map secondary_indices; // secondary indices fields + + string get_index_type(const string& index_name)const { + if( index_name == primary_index.name.to_string() ) return primary_index.type; + for( const auto& kv : secondary_indices ) { + if( index_name == kv.first.to_string() ) return kv.second.type; + } + return {}; + } + }; + struct clause_pair { clause_pair() = default; clause_pair( const string& id, const string& body ) @@ -95,31 +151,51 @@ struct variant_def { vector types; }; +struct action_result_def { + action_result_def() = default; + action_result_def(const action_name& name, const type_name& result_type) + :name(name), result_type(result_type) + {} + + action_name name; + type_name result_type; +}; + template struct may_not_exist { T value{}; }; +// Needed for custom from_variant and to_variant of kv_table_def +template +struct kv_tables_as_object { + T value{}; +}; + struct abi_def { abi_def() = default; - abi_def(const vector& types, const vector& structs, const vector& actions, const vector& tables, const vector& clauses, const vector& error_msgs) + abi_def(const vector& types, const vector& structs, const vector& actions, const vector& tables, const vector& clauses, const vector& error_msgs, const kv_tables_as_object>& kv_tables) :types(types) ,structs(structs) ,actions(actions) ,tables(tables) ,ricardian_clauses(clauses) ,error_messages(error_msgs) + ,kv_tables(kv_tables) + {} - string version = ""; - vector types; - vector structs; - vector actions; - vector tables; - vector ricardian_clauses; - vector error_messages; - extensions_type abi_extensions; - may_not_exist> variants; + string version = ""; + vector types; + vector structs; + vector actions; + vector tables; + vector ricardian_clauses; + vector error_messages; + extensions_type abi_extensions; + may_not_exist> variants; + may_not_exist> action_results; + kv_tables_as_object> kv_tables; }; abi_def eosio_contract_abi(const abi_def& eosio_system_abi); @@ -132,13 +208,13 @@ extern unsigned char eosio_abi_bin[2132]; namespace fc { template -datastream& operator << (datastream& s, const eosio::chain::may_not_exist& v) { +ST& operator << (ST& s, const eosio::chain::may_not_exist& v) { raw::pack(s, v.value); return s; } template -datastream& operator >> (datastream& s, eosio::chain::may_not_exist& v) { +ST& operator >> (ST& s, eosio::chain::may_not_exist& v) { if (s.remaining()) raw::unpack(s, v.value); return s; @@ -154,6 +230,67 @@ void from_variant(const fc::variant& v, eosio::chain::may_not_exist& e) { from_variant( v, e.value ); } + +template +ST& operator << (ST& s, const eosio::chain::kv_tables_as_object& v) { + raw::pack(s, v.value); + return s; +} + +template +ST& operator >> (ST& s, eosio::chain::kv_tables_as_object& v) { + if( s.remaining() ) + raw::unpack(s, v.value); + return s; +} + +template +void to_variant(const eosio::chain::kv_tables_as_object& o, fc::variant& v) { + const auto &kv_tables = o.value; + mutable_variant_object mvo; + + for( const auto &table : kv_tables ) { + mutable_variant_object vo_table; + + variant primary_index; + to_variant(table.second.primary_index, primary_index); + + mutable_variant_object secondary_indices; + for( const auto &sec_index : table.second.secondary_indices ) { + variant sidx; + to_variant(sec_index.second, sidx); + secondary_indices(sec_index.first.to_string(), sidx); + } + + vo_table("type", variant(table.second.type))("primary_index", primary_index)("secondary_indices", variant(secondary_indices)); + mvo(table.first.to_string(), variant(vo_table)); + } + v = variant(mvo); +} + +template +void from_variant(const fc::variant& v, eosio::chain::kv_tables_as_object& o) { + EOS_ASSERT( v.is_object(), eosio::chain::invalid_type_inside_abi, "variant is not an variant_object type"); + + auto &kv_tables = o.value; + const auto& tables = v.get_object(); + + for( const auto& table_it : tables ) { + const auto &table_obj = table_it.value().get_object(); + eosio::chain::kv_table_def kv_tbl_def; + from_variant(table_obj["type"], kv_tbl_def.type); + from_variant(table_obj["primary_index"], kv_tbl_def.primary_index); + if( const auto st_it = table_obj.find("secondary_indices"); st_it != table_obj.end() ) { + const auto &sec_indices_obj = st_it->value().get_object(); + for( const auto& sidx_it : sec_indices_obj ) { + eosio::chain::secondary_index_def idx_def; + from_variant(sidx_it.value(), idx_def); + kv_tbl_def.secondary_indices[eosio::chain::index_name(sidx_it.key())] = idx_def; + } + } + kv_tables[eosio::chain::name(table_it.key())] = kv_tbl_def; + } +} } // namespace fc FC_REFLECT( eosio::chain::type_def , (new_type_name)(type) ) @@ -161,8 +298,12 @@ FC_REFLECT( eosio::chain::field_def , (name)(type) ) FC_REFLECT( eosio::chain::struct_def , (name)(base)(fields) ) FC_REFLECT( eosio::chain::action_def , (name)(type)(ricardian_contract) ) FC_REFLECT( eosio::chain::table_def , (name)(index_type)(key_names)(key_types)(type) ) +FC_REFLECT( eosio::chain::primary_index_def , (name)(type) ) +FC_REFLECT( eosio::chain::secondary_index_def , (type) ) +FC_REFLECT( eosio::chain::kv_table_def , (type)(primary_index)(secondary_indices) ) FC_REFLECT( eosio::chain::clause_pair , (id)(body) ) FC_REFLECT( eosio::chain::error_message , (error_code)(error_msg) ) FC_REFLECT( eosio::chain::variant_def , (name)(types) ) +FC_REFLECT( eosio::chain::action_result_def , (name)(result_type) ) FC_REFLECT( eosio::chain::abi_def , (version)(types)(structs)(actions)(tables) - (ricardian_clauses)(error_messages)(abi_extensions)(variants) ) + (ricardian_clauses)(error_messages)(abi_extensions)(variants)(action_results)(kv_tables) ) diff --git a/libraries/chain/include/eosio/chain/abi_serializer.hpp b/libraries/chain/include/eosio/chain/abi_serializer.hpp index 1be1095c94e..bd4cc23f032 100644 --- a/libraries/chain/include/eosio/chain/abi_serializer.hpp +++ b/libraries/chain/include/eosio/chain/abi_serializer.hpp @@ -35,23 +35,18 @@ struct abi_serializer { abi_serializer(){ configure_built_in_types(); } abi_serializer( const abi_def& abi, const yield_function_t& yield ); - [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]] - abi_serializer( const abi_def& abi, const fc::microseconds& max_serialization_time ); void set_abi( const abi_def& abi, const yield_function_t& yield ); - [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]] - void set_abi(const abi_def& abi, const fc::microseconds& max_serialization_time); /// @return string_view of `t` or internal string type std::string_view resolve_type(const std::string_view& t)const; bool is_array(const std::string_view& type)const; bool is_optional(const std::string_view& type)const; bool is_type( const std::string_view& type, const yield_function_t& yield )const; - [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]] - bool is_type(const std::string_view& type, const fc::microseconds& max_serialization_time)const; bool is_builtin_type(const std::string_view& type)const; bool is_integer(const std::string_view& type) const; int get_integer_size(const std::string_view& type) const; bool is_struct(const std::string_view& type)const; + bool is_kv_table(const std::string_view& type)const; /// @return string_view of `type` std::string_view fundamental_type(const std::string_view& type)const; @@ -60,34 +55,22 @@ struct abi_serializer { type_name get_action_type(name action)const; type_name get_table_type(name action)const; + type_name get_kv_table_type(name action)const; + type_name get_action_result_type(name action_result)const; - optional get_error_message( uint64_t error_code )const; + std::optional get_error_message( uint64_t error_code )const; fc::variant binary_to_variant( const std::string_view& type, const bytes& binary, const yield_function_t& yield, bool short_path = false )const; - [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]] - fc::variant binary_to_variant( const std::string_view& type, const bytes& binary, const fc::microseconds& max_serialization_time, bool short_path = false )const; fc::variant binary_to_variant( const std::string_view& type, fc::datastream& binary, const yield_function_t& yield, bool short_path = false )const; - [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]] - fc::variant binary_to_variant( const std::string_view& type, fc::datastream& binary, const fc::microseconds& max_serialization_time, bool short_path = false )const; - [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]] - bytes variant_to_binary( const std::string_view& type, const fc::variant& var, const fc::microseconds& max_serialization_time, bool short_path = false )const; bytes variant_to_binary( const std::string_view& type, const fc::variant& var, const yield_function_t& yield, bool short_path = false )const; - [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]] - void variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream& ds, const fc::microseconds& max_serialization_time, bool short_path = false )const; void variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream& ds, const yield_function_t& yield, bool short_path = false )const; template static void to_variant( const T& o, fc::variant& vo, Resolver resolver, const yield_function_t& yield ); - template - [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]] - static void to_variant( const T& o, fc::variant& vo, Resolver resolver, const fc::microseconds& max_serialization_time ); template static void from_variant( const fc::variant& v, T& o, Resolver resolver, const yield_function_t& yield ); - template - [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]] - static void from_variant( const fc::variant& v, T& o, Resolver resolver, const fc::microseconds& max_serialization_time ); template static bool is_empty_abi(const Vec& abi_vec) @@ -137,8 +120,10 @@ struct abi_serializer { map> structs; map actions; map tables; + map> kv_tables; map error_messages; map> variants; + map action_results; map, std::less<>> built_in_types; void configure_built_in_types(); @@ -194,7 +179,7 @@ namespace impl { map::const_iterator variant_itr; }; - using path_root = static_variant; + using path_root = std::variant; struct empty_path_item {}; @@ -213,7 +198,7 @@ namespace impl { uint32_t variant_ordinal = 0; }; - using path_item = static_variant; + using path_item = std::variant; struct abi_traverse_context_with_path : public abi_traverse_context { abi_traverse_context_with_path( const abi_serializer& abis, abi_serializer::yield_function_t yield, const std::string_view& type ) @@ -274,11 +259,14 @@ namespace impl { template constexpr bool single_type_requires_abi_v() { return std::is_base_of::value || + std::is_same::value || std::is_same::value || std::is_same::value || + std::is_same::value || std::is_same::value || std::is_same::value || std::is_same::value || + std::is_same::value || std::is_same::value || std::is_same::value; } @@ -346,7 +334,7 @@ namespace impl { static void add( mutable_variant_object &mvo, const char* name, const vector& v, Resolver resolver, abi_traverse_context& ctx ) { auto h = ctx.enter_scope(); - vector array; + vector array; array.reserve(v.size()); for (const auto& iter: v) { @@ -357,6 +345,24 @@ namespace impl { mvo(name, std::move(array)); } + /** + * template which overloads add for deques of types which contain ABI information in their trees + * for these members we call ::add in order to trigger further processing + */ + template = 1> + static void add( mutable_variant_object &mvo, const char* name, const deque& v, Resolver resolver, abi_traverse_context& ctx ) + { + auto h = ctx.enter_scope(); + deque array; + + for (const auto& iter: v) { + mutable_variant_object elem_mvo; + add(elem_mvo, "_", iter, resolver, ctx); + array.emplace_back(std::move(elem_mvo["_"])); + } + mvo(name, std::move(array)); + } + /** * template which overloads add for shared_ptr of types which contain ABI information in their trees * for these members we call ::add in order to trigger further processing @@ -389,12 +395,12 @@ namespace impl { }; template - static void add( mutable_variant_object &mvo, const char* name, const fc::static_variant& v, Resolver resolver, abi_traverse_context& ctx ) + static void add( mutable_variant_object &mvo, const char* name, const std::variant& v, Resolver resolver, abi_traverse_context& ctx ) { auto h = ctx.enter_scope(); mutable_variant_object obj_mvo; add_static_variant adder(obj_mvo, resolver, ctx); - v.visit(adder); + std::visit(adder, v); mvo(name, std::move(obj_mvo["_"])); } @@ -419,7 +425,7 @@ namespace impl { try { auto abi = resolver(act.account); - if (abi.valid()) { + if (abi) { auto type = abi->get_action_type(act.name); if (!type.empty()) { try { @@ -444,21 +450,68 @@ namespace impl { } /** - * overload of to_variant_object for packed_transaction + * overload of to_variant_object for action_trace * - * This matches the FC_REFLECT for this type, but this is provided to allow extracting the contents of ptrx.transaction + * This matches the FC_REFLECT for this type, but this is provided to extract the contents of action_trace.return_value * @tparam Resolver - * @param act + * @param action_trace * @param resolver * @return */ template - static void add( mutable_variant_object &out, const char* name, const packed_transaction& ptrx, Resolver resolver, abi_traverse_context& ctx ) + static void add( mutable_variant_object& out, const char* name, const action_trace& act_trace, Resolver resolver, abi_traverse_context& ctx ) { - static_assert(fc::reflector::total_member_count == 4); + static_assert(fc::reflector::total_member_count == 18); auto h = ctx.enter_scope(); mutable_variant_object mvo; - auto trx = ptrx.get_transaction(); + + mvo("action_ordinal", act_trace.action_ordinal); + mvo("creator_action_ordinal", act_trace.creator_action_ordinal); + mvo("closest_unnotified_ancestor_action_ordinal", act_trace.closest_unnotified_ancestor_action_ordinal); + mvo("receipt", act_trace.receipt); + mvo("receiver", act_trace.receiver); + add(mvo, "act", act_trace.act, resolver, ctx); + mvo("context_free", act_trace.context_free); + mvo("elapsed", act_trace.elapsed); + mvo("console", act_trace.console); + mvo("trx_id", act_trace.trx_id); + mvo("block_num", act_trace.block_num); + mvo("block_time", act_trace.block_time); + mvo("producer_block_id", act_trace.producer_block_id); + mvo("account_ram_deltas", act_trace.account_ram_deltas); + mvo("account_disk_deltas", act_trace.account_disk_deltas); + mvo("except", act_trace.except); + mvo("error_code", act_trace.error_code); + + mvo("return_value_hex_data", act_trace.return_value); + auto act = act_trace.act; + try { + auto abi = resolver(act.account); + if (abi) { + auto type = abi->get_action_result_type(act.name); + if (!type.empty()) { + binary_to_variant_context _ctx(*abi, ctx, type); + _ctx.short_path = true; // Just to be safe while avoiding the complexity of threading an override boolean all over the place + mvo( "return_value_data", abi->_binary_to_variant( type, act_trace.return_value, _ctx )); + } + } + } catch(...) {} + out(name, std::move(mvo)); + } + + /** + * overload of to_variant_object for packed_transaction_v0 + * + * This matches the FC_REFLECT for packed_transaction_v0 type with the addition of "transaction" which is provided + * to allow extracting the contents of ptrx.get_transaction() + */ + template + static void add( mutable_variant_object& out, const char* name, const packed_transaction_v0& ptrx, Resolver resolver, abi_traverse_context& ctx ) + { + static_assert(fc::reflector::total_member_count == 4); + auto h = ctx.enter_scope(); + mutable_variant_object mvo; + const auto& trx = ptrx.get_transaction(); mvo("id", trx.id()); mvo("signatures", ptrx.get_signatures()); mvo("compression", ptrx.get_compression()); @@ -466,21 +519,47 @@ namespace impl { mvo("context_free_data", ptrx.get_context_free_data()); mvo("packed_trx", ptrx.get_packed_transaction()); add(mvo, "transaction", trx, resolver, ctx); - out(name, std::move(mvo)); } /** - * overload of to_variant_object for transaction + * overload of to_variant_object for packed_transaction, providing original packed_transaction_v0 variant layout * - * This matches the FC_REFLECT for this type, but this is provided to allow extracting the contents of trx.transaction_extensions + * This matches the FC_REFLECT for packed_transaction_v0 type with the addition of "transaction" which is provided + * to allow extracting the contents of ptrx.get_transaction(). The generated variant should match above method. */ template - static void add( mutable_variant_object &out, const char* name, const transaction& trx, Resolver resolver, abi_traverse_context& ctx ) + static void add( mutable_variant_object& out, const char* name, const packed_transaction& ptrx, Resolver resolver, abi_traverse_context& ctx ) { - static_assert(fc::reflector::total_member_count == 9); + static_assert(fc::reflector::total_member_count == 3); auto h = ctx.enter_scope(); mutable_variant_object mvo; + const auto& trx = ptrx.get_transaction(); + mvo("id", trx.id()); + const auto* sigs = ptrx.get_signatures(); + if( std::holds_alternative(ptrx.get_prunable_data().prunable_data) ) { + const auto& legacy = std::get(ptrx.get_prunable_data().prunable_data); + mvo("signatures", legacy.signatures ); + } else { + mvo("signatures", vector()); + } + mvo("compression", ptrx.get_compression()); + if( std::holds_alternative(ptrx.get_prunable_data().prunable_data) ) { + const auto& legacy = std::get(ptrx.get_prunable_data().prunable_data); + mvo("packed_context_free_data", legacy.packed_context_free_data); + mvo("context_free_data", legacy.context_free_segments); + } else { + mvo("packed_context_free_data", bytes()); + mvo("context_free_data", vector()); + } + mvo("packed_trx", ptrx.get_packed_transaction()); + add(mvo, "transaction", trx, resolver, ctx); + out(name, std::move(mvo)); + } + + template + static void add_transaction( mutable_variant_object& mvo, const transaction& trx, Resolver resolver, abi_traverse_context& ctx ) + { mvo("expiration", trx.expiration); mvo("ref_block_num", trx.ref_block_num); mvo("ref_block_prefix", trx.ref_block_prefix); @@ -493,23 +572,49 @@ namespace impl { // process contents of block.transaction_extensions auto exts = trx.validate_and_extract_extensions(); if (exts.count(deferred_transaction_generation_context::extension_id()) > 0) { - const auto& deferred_transaction_generation = exts.lower_bound(deferred_transaction_generation_context::extension_id())->second.get(); + const auto& deferred_transaction_generation = std::get(exts.lower_bound(deferred_transaction_generation_context::extension_id())->second); mvo("deferred_transaction_generation", deferred_transaction_generation); } + } + + /** + * overload of to_variant_object for transaction + * + * This matches the FC_REFLECT for this type, but this is provided to allow extracting the contents of trx.transaction_extensions + */ + template + static void add( mutable_variant_object &out, const char* name, const transaction& trx, Resolver resolver, abi_traverse_context& ctx ) + { + static_assert(fc::reflector::total_member_count == 9); + auto h = ctx.enter_scope(); + mutable_variant_object mvo; + add_transaction(mvo, trx, resolver, ctx); out(name, std::move(mvo)); } /** - * overload of to_variant_object for signed_block + * overload of to_variant_object for signed_transaction * - * This matches the FC_REFLECT for this type, but this is provided to allow extracting the contents of - * block.header_extensions and block.block_extensions + * This matches the FC_REFLECT for this type, but this is provided to allow extracting the contents of trx.transaction_extensions */ template - static void add( mutable_variant_object &out, const char* name, const signed_block& block, Resolver resolver, abi_traverse_context& ctx ) + static void add( mutable_variant_object &out, const char* name, const signed_transaction& trx, Resolver resolver, abi_traverse_context& ctx ) + { + static_assert(fc::reflector::total_member_count == 11); + auto h = ctx.enter_scope(); + mutable_variant_object mvo; + add_transaction(mvo, trx, resolver, ctx); + mvo("signatures", trx.signatures); + mvo("context_free_data", trx.context_free_data); + + out(name, std::move(mvo)); + } + + template + static void add_signed_block( mutable_variant_object& out, const char* name, const SignedBlock& block, Resolver resolver, abi_traverse_context& ctx ) { - static_assert(fc::reflector::total_member_count == 12); + static_assert( std::is_same_v || std::is_same_v ); auto h = ctx.enter_scope(); mutable_variant_object mvo; mvo("timestamp", block.timestamp); @@ -524,8 +629,9 @@ namespace impl { // process contents of block.header_extensions flat_multimap header_exts = block.validate_and_extract_header_extensions(); if ( header_exts.count(protocol_feature_activation::extension_id() > 0) ) { - const auto& new_protocol_features = header_exts.lower_bound(protocol_feature_activation::extension_id())->second.get().protocol_features; - vector pf_array; + const auto& new_protocol_features = + std::get(header_exts.lower_bound(protocol_feature_activation::extension_id())->second).protocol_features; + vector pf_array; pf_array.reserve(new_protocol_features.size()); for (auto feature : new_protocol_features) { mutable_variant_object feature_mvo; @@ -535,7 +641,8 @@ namespace impl { mvo("new_protocol_features", pf_array); } if ( header_exts.count(producer_schedule_change_extension::extension_id())) { - const auto& new_producer_schedule = header_exts.lower_bound(producer_schedule_change_extension::extension_id())->second.get(); + const auto& new_producer_schedule = + std::get(header_exts.lower_bound(producer_schedule_change_extension::extension_id())->second); mvo("new_producer_schedule", new_producer_schedule); } @@ -545,12 +652,36 @@ namespace impl { // process contents of block.block_extensions auto block_exts = block.validate_and_extract_extensions(); if ( block_exts.count(additional_block_signatures_extension::extension_id()) > 0) { - const auto& additional_signatures = block_exts.lower_bound(additional_block_signatures_extension::extension_id())->second.get(); + const auto& additional_signatures = + std::get(block_exts.lower_bound(additional_block_signatures_extension::extension_id())->second); mvo("additional_signatures", additional_signatures); } out(name, std::move(mvo)); } + + /** + * overload of to_variant_object for signed_block, support old signed_block_v0 format + */ + template + static void add( mutable_variant_object& out, const char* name, const signed_block& block, Resolver resolver, abi_traverse_context& ctx ) + { + static_assert(fc::reflector::total_member_count == 13); + add_signed_block( out, name, block, std::move(resolver), ctx ); + } + + /** + * overload of to_variant_object for signed_block_v0 + * + * This matches the FC_REFLECT for this type, but this is provided to allow extracting the contents of + * block.header_extensions and block.block_extensions + */ + template + static void add( mutable_variant_object &out, const char* name, const signed_block_v0& block, Resolver resolver, abi_traverse_context& ctx ) + { + static_assert(fc::reflector::total_member_count == 12); + add_signed_block( out, name, block, std::move(resolver), ctx ); + } }; /** @@ -558,7 +689,7 @@ namespace impl { * this will degrade to the common fc::to_variant as soon as the type no longer contains * ABI related info * - * @tparam Reslover - callable with the signature (const name& code_account) -> optional + * @tparam Reslover - callable with the signature (const name& code_account) -> std::optional */ template class abi_to_variant_visitor @@ -597,7 +728,7 @@ namespace impl { * and can be degraded to the normal ::from_variant(...) processing */ template = 1> - static void extract( const variant& v, M& o, Resolver, abi_traverse_context& ctx ) + static void extract( const fc::variant& v, M& o, Resolver, abi_traverse_context& ctx ) { auto h = ctx.enter_scope(); from_variant(v, o); @@ -608,14 +739,14 @@ namespace impl { * for these types we create new ABI aware visitors */ template = 1> - static void extract( const variant& v, M& o, Resolver resolver, abi_traverse_context& ctx ); + static void extract( const fc::variant& v, M& o, Resolver resolver, abi_traverse_context& ctx ); /** * template which overloads extract for vectors of types which contain ABI information in their trees * for these members we call ::extract in order to trigger further processing */ template = 1> - static void extract( const variant& v, vector& o, Resolver resolver, abi_traverse_context& ctx ) + static void extract( const fc::variant& v, vector& o, Resolver resolver, abi_traverse_context& ctx ) { auto h = ctx.enter_scope(); const variants& array = v.get_array(); @@ -628,12 +759,29 @@ namespace impl { } } + /** + * template which overloads extract for deque of types which contain ABI information in their trees + * for these members we call ::extract in order to trigger further processing + */ + template = 1> + static void extract( const fc::variant& v, deque& o, Resolver resolver, abi_traverse_context& ctx ) + { + auto h = ctx.enter_scope(); + const variants& array = v.get_array(); + o.clear(); + for( auto itr = array.begin(); itr != array.end(); ++itr ) { + M o_iter; + extract(*itr, o_iter, resolver, ctx); + o.emplace_back(std::move(o_iter)); + } + } + /** * template which overloads extract for shared_ptr of types which contain ABI information in their trees * for these members we call ::extract in order to trigger further processing */ template = 1> - static void extract( const variant& v, std::shared_ptr& o, Resolver resolver, abi_traverse_context& ctx ) + static void extract( const fc::variant& v, std::shared_ptr& o, Resolver resolver, abi_traverse_context& ctx ) { auto h = ctx.enter_scope(); const variant_object& vo = v.get_object(); @@ -648,7 +796,7 @@ namespace impl { * exploded and processed explicitly */ template - static void extract( const variant& v, action& act, Resolver resolver, abi_traverse_context& ctx ) + static void extract( const fc::variant& v, action& act, Resolver resolver, abi_traverse_context& ctx ) { auto h = ctx.enter_scope(); const variant_object& vo = v.get_object(); @@ -669,7 +817,7 @@ namespace impl { valid_empty_data = act.data.empty(); } else if ( data.is_object() ) { auto abi = resolver(act.account); - if (abi.valid()) { + if (abi) { auto type = abi->get_action_type(act.name); if (!type.empty()) { variant_to_binary_context _ctx(*abi, ctx, type); @@ -695,14 +843,91 @@ namespace impl { } template - static void extract( const variant& v, packed_transaction& ptrx, Resolver resolver, abi_traverse_context& ctx ) + static void extract_transaction( const variant_object& vo, transaction& trx, Resolver resolver, abi_traverse_context& ctx ) + { + if (vo.contains("expiration")) { + from_variant(vo["expiration"], trx.expiration); + } + if (vo.contains("ref_block_num")) { + from_variant(vo["ref_block_num"], trx.ref_block_num); + } + if (vo.contains("ref_block_prefix")) { + from_variant(vo["ref_block_prefix"], trx.ref_block_prefix); + } + if (vo.contains("max_net_usage_words")) { + from_variant(vo["max_net_usage_words"], trx.max_net_usage_words); + } + if (vo.contains("max_cpu_usage_ms")) { + from_variant(vo["max_cpu_usage_ms"], trx.max_cpu_usage_ms); + } + if (vo.contains("delay_sec")) { + from_variant(vo["delay_sec"], trx.delay_sec); + } + if (vo.contains("context_free_actions")) { + extract(vo["context_free_actions"], trx.context_free_actions, resolver, ctx); + } + if (vo.contains("actions")) { + extract(vo["actions"], trx.actions, resolver, ctx); + } + + // can have "deferred_transaction_generation" (if there is a deferred transaction and the extension was "extracted" to show data), + // or "transaction_extensions" (either as empty or containing the packed deferred transaction), + // or both (when there is a deferred transaction and extension was "extracted" to show data and a redundant "transaction_extensions" was provided), + // or neither (only if extension was "extracted" and there was no deferred transaction to extract) + if (vo.contains("deferred_transaction_generation")) { + deferred_transaction_generation_context deferred_transaction_generation; + from_variant(vo["deferred_transaction_generation"], deferred_transaction_generation); + emplace_extension( + trx.transaction_extensions, + deferred_transaction_generation_context::extension_id(), + fc::raw::pack( deferred_transaction_generation ) + ); + // if both are present, they need to match + if (vo.contains("transaction_extensions")) { + extensions_type trx_extensions; + from_variant(vo["transaction_extensions"], trx_extensions); + EOS_ASSERT(trx.transaction_extensions == trx_extensions, packed_transaction_type_exception, + "Transaction contained deferred_transaction_generation and transaction_extensions that did not match"); + } + } + else if (vo.contains("transaction_extensions")) { + from_variant(vo["transaction_extensions"], trx.transaction_extensions); + } + } + + template + static void extract( const fc::variant& v, transaction& trx, Resolver resolver, abi_traverse_context& ctx ) + { + static_assert(fc::reflector::total_member_count == 9); + auto h = ctx.enter_scope(); + const variant_object& vo = v.get_object(); + extract_transaction(vo, trx, resolver, ctx); + } + + template + static void extract( const fc::variant& v, signed_transaction& strx, Resolver resolver, abi_traverse_context& ctx ) + { + static_assert(fc::reflector::total_member_count == 11); + auto h = ctx.enter_scope(); + const variant_object& vo = v.get_object(); + extract_transaction(vo, strx, resolver, ctx); + if (vo.contains("signatures")) { + from_variant(vo["signatures"], strx.signatures); + } + if (vo.contains("context_free_data")) { + from_variant(vo["context_free_data"], strx.context_free_data); + } + } + + template + static void extract( const fc::variant& v, packed_transaction_v0& ptrx, Resolver resolver, abi_traverse_context& ctx ) { auto h = ctx.enter_scope(); const variant_object& vo = v.get_object(); EOS_ASSERT(vo.contains("signatures"), packed_transaction_type_exception, "Missing signatures"); EOS_ASSERT(vo.contains("compression"), packed_transaction_type_exception, "Missing compression"); std::vector signatures; - packed_transaction::compression_type compression; + packed_transaction_v0::compression_type compression; from_variant(vo["signatures"], signatures); from_variant(vo["compression"], compression); @@ -720,25 +945,26 @@ namespace impl { bytes packed_trx; from_variant(vo["packed_trx"], packed_trx); if( use_packed_cfd ) { - ptrx = packed_transaction( std::move( packed_trx ), std::move( signatures ), std::move( packed_cfd ), compression ); + ptrx = packed_transaction_v0( std::move( packed_trx ), std::move( signatures ), std::move( packed_cfd ), compression ); } else { - ptrx = packed_transaction( std::move( packed_trx ), std::move( signatures ), std::move( cfd ), compression ); + ptrx = packed_transaction_v0( std::move( packed_trx ), std::move( signatures ), std::move( cfd ), compression ); } } else { EOS_ASSERT(vo.contains("transaction"), packed_transaction_type_exception, "Missing transaction"); if( use_packed_cfd ) { transaction trx; extract( vo["transaction"], trx, resolver, ctx ); - ptrx = packed_transaction( std::move(trx), std::move(signatures), std::move(packed_cfd), compression ); + ptrx = packed_transaction_v0( std::move(trx), std::move(signatures), std::move(packed_cfd), compression ); } else { signed_transaction trx; extract( vo["transaction"], trx, resolver, ctx ); trx.signatures = std::move( signatures ); trx.context_free_data = std::move(cfd); - ptrx = packed_transaction( std::move( trx ), compression ); + ptrx = packed_transaction_v0( std::move( trx ), compression ); } } } + }; /** @@ -746,7 +972,7 @@ namespace impl { * this will degrade to the common fc::from_variant as soon as the type no longer contains * ABI related info * - * @tparam Reslover - callable with the signature (const name& code_account) -> optional + * @tparam Reslover - callable with the signature (const name& code_account) -> std::optional */ template class abi_from_variant_visitor : public reflector_init_visitor @@ -790,7 +1016,7 @@ namespace impl { } template> - void abi_from_variant::extract( const variant& v, M& o, Resolver resolver, abi_traverse_context& ctx ) + void abi_from_variant::extract( const fc::variant& v, M& o, Resolver resolver, abi_traverse_context& ctx ) { auto h = ctx.enter_scope(); const variant_object& vo = v.get_object(); @@ -799,7 +1025,7 @@ namespace impl { } /// namespace eosio::chain::impl template -void abi_serializer::to_variant( const T& o, variant& vo, Resolver resolver, const yield_function_t& yield ) try { +void abi_serializer::to_variant( const T& o, fc::variant& vo, Resolver resolver, const yield_function_t& yield ) try { mutable_variant_object mvo; impl::abi_traverse_context ctx( yield ); impl::abi_to_variant::add(mvo, "_", o, resolver, ctx); @@ -807,20 +1033,11 @@ void abi_serializer::to_variant( const T& o, variant& vo, Resolver resolver, con } FC_RETHROW_EXCEPTIONS(error, "Failed to serialize: ${type}", ("type", boost::core::demangle( typeid(o).name() ) )) template -void abi_serializer::to_variant( const T& o, variant& vo, Resolver resolver, const fc::microseconds& max_serialization_time ) { - to_variant( o, vo, resolver, create_yield_function(max_serialization_time) ); -} - -template -void abi_serializer::from_variant( const variant& v, T& o, Resolver resolver, const yield_function_t& yield ) try { +void abi_serializer::from_variant( const fc::variant& v, T& o, Resolver resolver, const yield_function_t& yield ) try { + static_assert( !std::is_same_v, "use packed_transaction_v0" ); + static_assert( !std::is_same_v, "use signed_block_v0" ); impl::abi_traverse_context ctx( yield ); impl::abi_from_variant::extract(v, o, resolver, ctx); } FC_RETHROW_EXCEPTIONS(error, "Failed to deserialize variant", ("variant",v)) -template -void abi_serializer::from_variant( const variant& v, T& o, Resolver resolver, const fc::microseconds& max_serialization_time ) { - from_variant( v, o, resolver, create_yield_function(max_serialization_time) ); -} - - } } // eosio::chain diff --git a/libraries/chain/include/eosio/chain/account_object.hpp b/libraries/chain/include/eosio/chain/account_object.hpp index 8c8da161a24..d94149e33ea 100644 --- a/libraries/chain/include/eosio/chain/account_object.hpp +++ b/libraries/chain/include/eosio/chain/account_object.hpp @@ -18,9 +18,10 @@ namespace eosio { namespace chain { shared_blob abi; void set_abi( const eosio::chain::abi_def& a ) { - abi.resize( fc::raw::pack_size( a ) ); - fc::datastream ds( abi.data(), abi.size() ); - fc::raw::pack( ds, a ); + abi.resize_and_fill( fc::raw::pack_size( a ), [&a](char* data, std::size_t size) { + fc::datastream ds( data, size ); + fc::raw::pack( ds, a ); + }); } eosio::chain::abi_def get_abi()const { @@ -103,7 +104,6 @@ CHAINBASE_SET_INDEX_TYPE(eosio::chain::account_object, eosio::chain::account_ind CHAINBASE_SET_INDEX_TYPE(eosio::chain::account_metadata_object, eosio::chain::account_metadata_index) CHAINBASE_SET_INDEX_TYPE(eosio::chain::account_ram_correction_object, eosio::chain::account_ram_correction_index) - FC_REFLECT(eosio::chain::account_object, (name)(creation_date)(abi)) FC_REFLECT(eosio::chain::account_metadata_object, (name)(recv_sequence)(auth_sequence)(code_sequence)(abi_sequence) (code_hash)(last_code_update)(flags)(vm_type)(vm_version)) diff --git a/libraries/chain/include/eosio/chain/action.hpp b/libraries/chain/include/eosio/chain/action.hpp index 1e593447955..a3c6ed604ae 100644 --- a/libraries/chain/include/eosio/chain/action.hpp +++ b/libraries/chain/include/eosio/chain/action.hpp @@ -53,32 +53,38 @@ namespace eosio { namespace chain { * application code. An application code will check to see if the required authorization * were properly declared when it executes. */ - struct action { - account_name account; - action_name name; - vector authorization; + struct action_base { + account_name account; + action_name name; + vector authorization; + + action_base() = default; + + action_base( account_name acnt, action_name act, const vector& auth ) + : account(acnt), name(act), authorization(auth) {} + action_base( account_name acnt, action_name act, vector&& auth ) + : account(acnt), name(act), authorization(std::move(auth)) {} + }; + + struct action : public action_base { bytes data; - action(){} + action() = default; template::value, int> = 1> - action( vector auth, const T& value ) { - account = T::get_account(); - name = T::get_name(); - authorization = move(auth); + action( vector auth, const T& value ) + : action_base( T::get_account(), T::get_name(), std::move(auth) ) { data.assign(value.data(), value.data() + value.size()); } template::value, int> = 1> - action( vector auth, const T& value ) { - account = T::get_account(); - name = T::get_name(); - authorization = move(auth); - data = fc::raw::pack(value); + action( vector auth, const T& value ) + : action_base( T::get_account(), T::get_name(), std::move(auth) ) { + data = fc::raw::pack(value); } action( vector auth, account_name account, action_name name, const bytes& data ) - : account(account), name(name), authorization(move(auth)), data(data) { + : action_base(account, name, std::move(auth)), data(data) { } template @@ -89,6 +95,38 @@ namespace eosio { namespace chain { } }; + template + auto generate_action_digest(Hasher&& hash, const action& act, const vector& action_output) { + using hash_type = decltype(hash(nullptr, 0)); + hash_type hashes[2]; + const action_base* base = &act; + const auto action_base_size = fc::raw::pack_size(*base); + const auto action_input_size = fc::raw::pack_size(act.data); + const auto action_output_size = fc::raw::pack_size(action_output); + const auto rhs_size = action_input_size + action_output_size; + std::vector buff; + buff.reserve(std::max(action_base_size, rhs_size)); + { + buff.resize(action_base_size); + fc::datastream ds(buff.data(), action_base_size); + fc::raw::pack(ds, *base); + hashes[0] = hash(buff.data(), action_base_size); + } + { + buff.resize(rhs_size); + fc::datastream ds(buff.data(), rhs_size); + fc::raw::pack(ds, act.data); + fc::raw::pack(ds, action_output); + hashes[1] = hash(buff.data(), rhs_size); + } + auto hashes_size = fc::raw::pack_size(hashes[0]) + fc::raw::pack_size(hashes[1]); + buff.resize(hashes_size); // may cause reallocation but in practice will be unlikely + fc::datastream ds(buff.data(), hashes_size); + fc::raw::pack(ds, hashes[0]); + fc::raw::pack(ds, hashes[1]); + return hash(buff.data(), hashes_size); + } + struct action_notice : public action { account_name receiver; }; @@ -96,4 +134,5 @@ namespace eosio { namespace chain { } } /// namespace eosio::chain FC_REFLECT( eosio::chain::permission_level, (actor)(permission) ) -FC_REFLECT( eosio::chain::action, (account)(name)(authorization)(data) ) +FC_REFLECT( eosio::chain::action_base, (account)(name)(authorization) ) +FC_REFLECT_DERIVED( eosio::chain::action, (eosio::chain::action_base), (data) ) diff --git a/libraries/chain/include/eosio/chain/action_receipt.hpp b/libraries/chain/include/eosio/chain/action_receipt.hpp index 78ef25c7c00..dec17c8fcac 100644 --- a/libraries/chain/include/eosio/chain/action_receipt.hpp +++ b/libraries/chain/include/eosio/chain/action_receipt.hpp @@ -16,9 +16,20 @@ namespace eosio { namespace chain { fc::unsigned_int code_sequence = 0; ///< total number of setcodes fc::unsigned_int abi_sequence = 0; ///< total number of setabis - digest_type digest()const { return digest_type::hash(*this); } + digest_type digest()const { + digest_type::encoder e; + fc::raw::pack(e, receiver); + fc::raw::pack(e, act_digest); + fc::raw::pack(e, global_sequence); + fc::raw::pack(e, recv_sequence); + fc::raw::pack(e, auth_sequence); + fc::raw::pack(e, code_sequence); + fc::raw::pack(e, abi_sequence); + return e.result(); + } }; } } /// namespace eosio::chain -FC_REFLECT( eosio::chain::action_receipt, (receiver)(act_digest)(global_sequence)(recv_sequence)(auth_sequence)(code_sequence)(abi_sequence) ) +FC_REFLECT( eosio::chain::action_receipt, + (receiver)(act_digest)(global_sequence)(recv_sequence)(auth_sequence)(code_sequence)(abi_sequence) ) diff --git a/libraries/chain/include/eosio/chain/apply_context.hpp b/libraries/chain/include/eosio/chain/apply_context.hpp index 3a5889248bf..4a61e8297ed 100644 --- a/libraries/chain/include/eosio/chain/apply_context.hpp +++ b/libraries/chain/include/eosio/chain/apply_context.hpp @@ -2,6 +2,10 @@ #include #include #include +#include +#include +#include +#include #include #include #include @@ -15,148 +19,6 @@ class controller; class transaction_context; class apply_context { - private: - template - class iterator_cache { - public: - iterator_cache(){ - _end_iterator_to_table.reserve(8); - _iterator_to_object.reserve(32); - } - - /// Returns end iterator of the table. - int cache_table( const table_id_object& tobj ) { - auto itr = _table_cache.find(tobj.id); - if( itr != _table_cache.end() ) - return itr->second.second; - - auto ei = index_to_end_iterator(_end_iterator_to_table.size()); - _end_iterator_to_table.push_back( &tobj ); - _table_cache.emplace( tobj.id, make_pair(&tobj, ei) ); - return ei; - } - - const table_id_object& get_table( table_id_object::id_type i )const { - auto itr = _table_cache.find(i); - EOS_ASSERT( itr != _table_cache.end(), table_not_in_cache, "an invariant was broken, table should be in cache" ); - return *itr->second.first; - } - - int get_end_iterator_by_table_id( table_id_object::id_type i )const { - auto itr = _table_cache.find(i); - EOS_ASSERT( itr != _table_cache.end(), table_not_in_cache, "an invariant was broken, table should be in cache" ); - return itr->second.second; - } - - const table_id_object* find_table_by_end_iterator( int ei )const { - EOS_ASSERT( ei < -1, invalid_table_iterator, "not an end iterator" ); - auto indx = end_iterator_to_index(ei); - if( indx >= _end_iterator_to_table.size() ) return nullptr; - return _end_iterator_to_table[indx]; - } - - const T& get( int iterator ) { - EOS_ASSERT( iterator != -1, invalid_table_iterator, "invalid iterator" ); - EOS_ASSERT( iterator >= 0, table_operation_not_permitted, "dereference of end iterator" ); - EOS_ASSERT( (size_t)iterator < _iterator_to_object.size(), invalid_table_iterator, "iterator out of range" ); - auto result = _iterator_to_object[iterator]; - EOS_ASSERT( result, table_operation_not_permitted, "dereference of deleted object" ); - return *result; - } - - void remove( int iterator ) { - EOS_ASSERT( iterator != -1, invalid_table_iterator, "invalid iterator" ); - EOS_ASSERT( iterator >= 0, table_operation_not_permitted, "cannot call remove on end iterators" ); - EOS_ASSERT( (size_t)iterator < _iterator_to_object.size(), invalid_table_iterator, "iterator out of range" ); - - auto obj_ptr = _iterator_to_object[iterator]; - if( !obj_ptr ) return; - _iterator_to_object[iterator] = nullptr; - _object_to_iterator.erase( obj_ptr ); - } - - int add( const T& obj ) { - auto itr = _object_to_iterator.find( &obj ); - if( itr != _object_to_iterator.end() ) - return itr->second; - - _iterator_to_object.push_back( &obj ); - _object_to_iterator[&obj] = _iterator_to_object.size() - 1; - - return _iterator_to_object.size() - 1; - } - - private: - map> _table_cache; - vector _end_iterator_to_table; - vector _iterator_to_object; - map _object_to_iterator; - - /// Precondition: std::numeric_limits::min() < ei < -1 - /// Iterator of -1 is reserved for invalid iterators (i.e. when the appropriate table has not yet been created). - inline size_t end_iterator_to_index( int ei )const { return (-ei - 2); } - /// Precondition: indx < _end_iterator_to_table.size() <= std::numeric_limits::max() - inline int index_to_end_iterator( size_t indx )const { return -(indx + 2); } - }; /// class iterator_cache - - template - struct array_size; - - template - struct array_size< std::array > { - static constexpr size_t size = N; - }; - - template - class secondary_key_helper; - - template - class secondary_key_helper::type>::value>::type > - { - public: - typedef SecondaryKey secondary_key_type; - - static void set(secondary_key_type& sk_in_table, const secondary_key_type& sk_from_wasm) { - sk_in_table = sk_from_wasm; - } - - static void get(secondary_key_type& sk_from_wasm, const secondary_key_type& sk_in_table ) { - sk_from_wasm = sk_in_table; - } - - static auto create_tuple(const table_id_object& tab, const secondary_key_type& secondary) { - return boost::make_tuple( tab.id, secondary ); - } - }; - - template - class secondary_key_helper::type>::value && - std::is_pointer::type>::value>::type > - { - public: - typedef SecondaryKey secondary_key_type; - typedef SecondaryKeyProxy secondary_key_proxy_type; - typedef SecondaryKeyProxyConst secondary_key_proxy_const_type; - - static constexpr size_t N = array_size::size; - - static void set(secondary_key_type& sk_in_table, secondary_key_proxy_const_type sk_from_wasm) { - std::copy(sk_from_wasm, sk_from_wasm + N, sk_in_table.begin()); - } - - static void get(secondary_key_proxy_type sk_from_wasm, const secondary_key_type& sk_in_table) { - std::copy(sk_in_table.begin(), sk_in_table.end(), sk_from_wasm); - } - - static auto create_tuple(const table_id_object& tab, secondary_key_proxy_const_type sk_from_wasm) { - secondary_key_type secondary; - std::copy(sk_from_wasm, sk_from_wasm + N, secondary.begin()); - return boost::make_tuple( tab.id, secondary ); - } - }; - public: template::type, @@ -169,7 +31,7 @@ class apply_context { typedef SecondaryKeyProxy secondary_key_proxy_type; typedef SecondaryKeyProxyConst secondary_key_proxy_const_type; - using secondary_key_helper_t = secondary_key_helper; + using secondary_key_helper_t = backing_store::db_secondary_key_helper; generic_index( apply_context& c ):context(c){} @@ -189,11 +51,16 @@ class apply_context { o.payer = payer; }); + std::string event_id; context.db.modify( tab, [&]( auto& t ) { ++t.count; + + if (context.control.get_deep_mind_logger() != nullptr) { + event_id = backing_store::db_context::table_event(t.code, t.scope, t.table, name(id)); + } }); - context.update_db_usage( payer, config::billable_size_v ); + context.update_db_usage( payer, config::billable_size_v, backing_store::db_context::secondary_add_trace(context.get_action_id(), std::move(event_id)) ); itr_cache.cache_table( tab ); return itr_cache.add( obj ); @@ -201,11 +68,17 @@ class apply_context { void remove( int iterator ) { const auto& obj = itr_cache.get( iterator ); - context.update_db_usage( obj.payer, -( config::billable_size_v ) ); const auto& table_obj = itr_cache.get_table( obj.t_id ); EOS_ASSERT( table_obj.code == context.receiver, table_access_violation, "db access violation" ); + std::string event_id; + if (context.control.get_deep_mind_logger() != nullptr) { + event_id = backing_store::db_context::table_event(table_obj.code, table_obj.scope, table_obj.table, name(obj.primary_key)); + } + + context.update_db_usage( obj.payer, -( config::billable_size_v ), backing_store::db_context::secondary_rem_trace(context.get_action_id(), std::move(event_id)) ); + // context.require_write_lock( table_obj.scope ); context.db.modify( table_obj, [&]( auto& t ) { @@ -232,9 +105,14 @@ class apply_context { int64_t billing_size = config::billable_size_v; + std::string event_id; + if (context.control.get_deep_mind_logger() != nullptr) { + event_id = backing_store::db_context::table_event(table_obj.code, table_obj.scope, table_obj.table, name(obj.primary_key)); + } + if( obj.payer != payer ) { - context.update_db_usage( obj.payer, -(billing_size) ); - context.update_db_usage( payer, +(billing_size) ); + context.update_db_usage( obj.payer, -(billing_size), backing_store::db_context::secondary_update_rem_trace(context.get_action_id(), std::string(event_id)) ); + context.update_db_usage( payer, +(billing_size), backing_store::db_context::secondary_update_add_trace(context.get_action_id(), std::move(event_id)) ); } context.db.modify( obj, [&]( auto& o ) { @@ -441,8 +319,8 @@ class apply_context { } private: - apply_context& context; - iterator_cache itr_cache; + apply_context& context; + backing_store::db_chainbase_iter_store itr_cache; }; /// class generic_index @@ -465,6 +343,9 @@ class apply_context { uint32_t schedule_action( uint32_t ordinal_of_action_to_schedule, account_name receiver, bool context_free ); uint32_t schedule_action( action&& act_to_schedule, account_name receiver, bool context_free ); + private: + template + void check_unprivileged_resource_usage(const char* resource, const flat_set& deltas); /// Authorization methods: public: @@ -478,9 +359,9 @@ class apply_context { * * @throws missing_auth_exception If no sufficient permission was found */ - void require_authorization(const account_name& account); + void require_authorization(const account_name& account) const; bool has_authorization(const account_name& account) const; - void require_authorization(const account_name& account, const permission_name& permission); + void require_authorization(const account_name& account, const permission_name& permission) const; /** * @return true if account exists, false if it does not @@ -501,25 +382,28 @@ class apply_context { /// Console methods: public: - void console_append( const string& val ) { + void console_append( std::string_view val ) { _pending_console_output += val; } /// Database methods: public: - void update_db_usage( const account_name& payer, int64_t delta ); + void update_db_usage( const account_name& payer, int64_t delta, const storage_usage_trace& trace ); + + int db_store_i64_chainbase( name scope, name table, const account_name& payer, uint64_t id, const char* buffer, size_t buffer_size ); + void db_update_i64_chainbase( int iterator, account_name payer, const char* buffer, size_t buffer_size ); + void db_remove_i64_chainbase( int iterator ); + int db_get_i64_chainbase( int iterator, char* buffer, size_t buffer_size ); + int db_next_i64_chainbase( int iterator, uint64_t& primary ); + int db_previous_i64_chainbase( int iterator, uint64_t& primary ); + int db_find_i64_chainbase( name code, name scope, name table, uint64_t id ); + int db_lowerbound_i64_chainbase( name code, name scope, name table, uint64_t id ); + int db_upperbound_i64_chainbase( name code, name scope, name table, uint64_t id ); + int db_end_i64_chainbase( name code, name scope, name table ); - int db_store_i64( name scope, name table, const account_name& payer, uint64_t id, const char* buffer, size_t buffer_size ); - void db_update_i64( int iterator, account_name payer, const char* buffer, size_t buffer_size ); - void db_remove_i64( int iterator ); - int db_get_i64( int iterator, char* buffer, size_t buffer_size ); - int db_next_i64( int iterator, uint64_t& primary ); - int db_previous_i64( int iterator, uint64_t& primary ); - int db_find_i64( name code, name scope, name table, uint64_t id ); - int db_lowerbound_i64( name code, name scope, name table, uint64_t id ); - int db_upperbound_i64( name code, name scope, name table, uint64_t id ); - int db_end_i64( name code, name scope, name table ); +# warning look into if we can make any of the db_** methods and idx***'s methods const and provide a const interface + backing_store::db_context& db_get_context(); private: @@ -527,8 +411,30 @@ class apply_context { const table_id_object& find_or_create_table( name code, name scope, name table, const account_name &payer ); void remove_table( const table_id_object& tid ); - int db_store_i64( name code, name scope, name table, const account_name& payer, uint64_t id, const char* buffer, size_t buffer_size ); + /// KV Database methods: + public: + int64_t kv_erase(uint64_t contract, const char* key, uint32_t key_size); + int64_t kv_set(uint64_t contract, const char* key, uint32_t key_size, const char* value, uint32_t value_size, account_name payer); + bool kv_get(uint64_t contract, const char* key, uint32_t key_size, uint32_t& value_size); + uint32_t kv_get_data(uint32_t offset, char* data, uint32_t data_size); + uint32_t kv_it_create(uint64_t contract, const char* prefix, uint32_t size); + void kv_it_destroy(uint32_t itr); + int32_t kv_it_status(uint32_t itr); + int32_t kv_it_compare(uint32_t itr_a, uint32_t itr_b); + int32_t kv_it_key_compare(uint32_t itr, const char* key, uint32_t size); + int32_t kv_it_move_to_end(uint32_t itr); + int32_t kv_it_next(uint32_t itr, uint32_t* found_key_size, uint32_t* found_value_size); + int32_t kv_it_prev(uint32_t itr, uint32_t* found_key_size, uint32_t* found_value_size); + int32_t kv_it_lower_bound(uint32_t itr, const char* key, uint32_t size, uint32_t* found_key_size, uint32_t* found_value_size); + int32_t kv_it_key(uint32_t itr, uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size); + int32_t kv_it_value(uint32_t itr, uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size); + kv_context& kv_get_backing_store() { + EOS_ASSERT( kv_backing_store, action_validate_exception, "KV APIs cannot access state (null backing_store)" ); + return *kv_backing_store; + } + private: + void kv_check_iterator(uint32_t itr); /// Misc methods: public: @@ -537,13 +443,13 @@ class apply_context { int get_action( uint32_t type, uint32_t index, char* buffer, size_t buffer_size )const; int get_context_free_data( uint32_t index, char* buffer, size_t buffer_size )const; vector get_active_producers() const; - bytes get_packed_transaction(); uint64_t next_global_sequence(); uint64_t next_recv_sequence( const account_metadata_object& receiver_account ); uint64_t next_auth_sequence( account_name actor ); - void add_ram_usage( account_name account, int64_t ram_delta ); + void add_ram_usage( account_name account, int64_t ram_delta, const storage_usage_trace& trace ); + void finalize_trace( action_trace& trace, const fc::time_point& start ); bool is_context_free()const { return context_free; } @@ -553,6 +459,9 @@ class apply_context { action_name get_sender() const; + uint32_t get_action_id() const; + void increment_action_id(); + /// Fields: public: @@ -571,26 +480,29 @@ class apply_context { bool context_free = false; public: + std::vector action_return_value; generic_index idx64; generic_index idx128; generic_index idx256; generic_index idx_double; generic_index idx_long_double; + std::unique_ptr kv_backing_store; + std::vector> kv_iterators; + std::vector kv_destroyed_iterators; + private: - iterator_cache keyval_cache; - vector< std::pair > _notified; ///< keeps track of new accounts to be notifed of current message - vector _inline_actions; ///< action_ordinals of queued inline actions - vector _cfa_inline_actions; ///< action_ordinals of queued inline context-free actions - std::string _pending_console_output; - flat_set _account_ram_deltas; ///< flat_set of account_delta so json is an array of objects + backing_store::db_chainbase_iter_store db_iter_store; + vector< std::pair > _notified; ///< keeps track of new accounts to be notifed of current message + vector _inline_actions; ///< action_ordinals of queued inline actions + vector _cfa_inline_actions; ///< action_ordinals of queued inline context-free actions + std::string _pending_console_output; + flat_set _account_ram_deltas; ///< flat_set of account_delta so json is an array of objects - //bytes _cached_trx; + std::unique_ptr _db_context; }; using apply_handler = std::function; } } // namespace eosio::chain - -//FC_REFLECT(eosio::chain::apply_context::apply_results, (applied_actions)(deferred_transaction_requests)(deferred_transactions_count)) diff --git a/libraries/chain/include/eosio/chain/authority.hpp b/libraries/chain/include/eosio/chain/authority.hpp index b635f3942a4..420c9299761 100644 --- a/libraries/chain/include/eosio/chain/authority.hpp +++ b/libraries/chain/include/eosio/chain/authority.hpp @@ -7,7 +7,7 @@ namespace eosio { namespace chain { -using shared_public_key_data = fc::static_variant; +using shared_public_key_data = std::variant; struct shared_public_key { shared_public_key( shared_public_key_data&& p ) : @@ -15,17 +15,17 @@ struct shared_public_key { operator public_key_type() const { fc::crypto::public_key::storage_type public_key_storage; - pubkey.visit(overloaded { + std::visit(overloaded { [&](const auto& k1r1) { public_key_storage = k1r1; }, [&](const shared_string& wa) { - fc::datastream ds(wa.data(), wa.size()); + fc::datastream ds(wa.data(), wa.size()); fc::crypto::webauthn::public_key pub; fc::raw::unpack(ds, pub); public_key_storage = pub; } - }); + }, pubkey); return std::move(public_key_storage); } @@ -36,40 +36,40 @@ struct shared_public_key { shared_public_key_data pubkey; friend bool operator == ( const shared_public_key& lhs, const shared_public_key& rhs ) { - if(lhs.pubkey.which() != rhs.pubkey.which()) + if(lhs.pubkey.index() != rhs.pubkey.index()) return false; - return lhs.pubkey.visit(overloaded { + return std::visit(overloaded { [&](const fc::ecc::public_key_shim& k1) { - return k1._data == rhs.pubkey.get()._data; + return k1._data == std::get(rhs.pubkey)._data; }, [&](const fc::crypto::r1::public_key_shim& r1) { - return r1._data == rhs.pubkey.get()._data; + return r1._data == std::get(rhs.pubkey)._data; }, [&](const shared_string& wa) { - return wa == rhs.pubkey.get(); + return wa == std::get(rhs.pubkey); } - }); + }, lhs.pubkey); } friend bool operator==(const shared_public_key& l, const public_key_type& r) { - if(l.pubkey.which() != r._storage.which()) + if(l.pubkey.index() != r._storage.index()) return false; - return l.pubkey.visit(overloaded { + return std::visit(overloaded { [&](const fc::ecc::public_key_shim& k1) { - return k1._data == r._storage.get()._data; + return k1._data == std::get(r._storage)._data; }, [&](const fc::crypto::r1::public_key_shim& r1) { - return r1._data == r._storage.get()._data; + return r1._data == std::get(r._storage)._data; }, [&](const shared_string& wa) { - fc::datastream ds(wa.data(), wa.size()); + fc::datastream ds(wa.data(), wa.size()); fc::crypto::webauthn::public_key pub; fc::raw::unpack(ds, pub); - return pub == r._storage.get(); + return pub == std::get(r._storage); } - }); + }, l.pubkey); } friend bool operator==(const public_key_type& l, const shared_public_key& r) { @@ -105,19 +105,21 @@ struct shared_key_weight { } static shared_key_weight convert(chainbase::allocator allocator, const key_weight& k) { - return k.key._storage.visit(overloaded { + return std::visit(overloaded { [&](const auto& k1r1) { return shared_key_weight(k1r1, k.weight); }, [&](const fc::crypto::webauthn::public_key& wa) { size_t psz = fc::raw::pack_size(wa); - shared_string wa_ss(psz, boost::container::default_init, std::move(allocator)); - fc::datastream ds(wa_ss.data(), wa_ss.size()); - fc::raw::pack(ds, wa); + shared_string wa_ss(std::move(allocator)); + wa_ss.resize_and_fill( psz, [&wa]( char* data, std::size_t sz ) { + fc::datastream ds(data, sz); + fc::raw::pack(ds, wa); + }); return shared_key_weight(std::move(wa_ss), k.weight); } - }); + }, k.key._storage); } shared_public_key key; @@ -299,6 +301,9 @@ inline bool validate( const Authority& auth ) { } } // namespace eosio::chain +namespace fc { + void to_variant(const eosio::chain::shared_public_key& var, fc::variant& vo); +} // namespace fc FC_REFLECT(eosio::chain::permission_level_weight, (permission)(weight) ) FC_REFLECT(eosio::chain::key_weight, (key)(weight) ) @@ -306,4 +311,4 @@ FC_REFLECT(eosio::chain::wait_weight, (wait_sec)(weight) ) FC_REFLECT(eosio::chain::authority, (threshold)(keys)(accounts)(waits) ) FC_REFLECT(eosio::chain::shared_key_weight, (key)(weight) ) FC_REFLECT(eosio::chain::shared_authority, (threshold)(keys)(accounts)(waits) ) -FC_REFLECT(eosio::chain::shared_public_key, (pubkey)) \ No newline at end of file +FC_REFLECT(eosio::chain::shared_public_key, (pubkey)) diff --git a/libraries/chain/include/eosio/chain/authority_checker.hpp b/libraries/chain/include/eosio/chain/authority_checker.hpp index 6a741ab75c4..16a3f954090 100644 --- a/libraries/chain/include/eosio/chain/authority_checker.hpp +++ b/libraries/chain/include/eosio/chain/authority_checker.hpp @@ -129,7 +129,7 @@ namespace detail { return {range.begin(), range.end()}; } - static optional + static std::optional permission_status_in_cache( const permission_cache_type& permissions, const permission_level& level ) { @@ -141,7 +141,7 @@ namespace detail { if( itr != permissions.end() ) return itr->second; - return optional(); + return std::optional(); } private: @@ -210,7 +210,7 @@ namespace detail { template>> uint32_t operator()(const KeyWeight& permission) { - auto itr = boost::find( checker.provided_keys, permission.key ); + auto itr = boost::range::find( checker.provided_keys, permission.key ); if( itr != checker.provided_keys.end() ) { checker._used_keys[itr - checker.provided_keys.begin()] = true; total_weight += permission.weight; diff --git a/libraries/chain/include/eosio/chain/authorization_manager.hpp b/libraries/chain/include/eosio/chain/authorization_manager.hpp index 2d67cc7adab..a61770c3fdb 100644 --- a/libraries/chain/include/eosio/chain/authorization_manager.hpp +++ b/libraries/chain/include/eosio/chain/authorization_manager.hpp @@ -31,6 +31,7 @@ namespace eosio { namespace chain { permission_name name, permission_id_type parent, const authority& auth, + uint32_t action_id, time_point initial_creation_time = time_point() ); @@ -38,12 +39,13 @@ namespace eosio { namespace chain { permission_name name, permission_id_type parent, authority&& auth, + uint32_t action_id, time_point initial_creation_time = time_point() ); - void modify_permission( const permission_object& permission, const authority& auth ); + void modify_permission( const permission_object& permission, const authority& auth, uint32_t action_id ); - void remove_permission( const permission_object& permission ); + void remove_permission( const permission_object& permission, uint32_t action_id ); void update_permission_usage( const permission_object& permission ); @@ -59,10 +61,10 @@ namespace eosio { namespace chain { * @param code_account The account which publishes the contract that handles the message * @param type The type of message */ - optional lookup_minimum_permission( account_name authorizer_account, - scope_name code_account, - action_name type - )const; + std::optional lookup_minimum_permission( account_name authorizer_account, + scope_name code_account, + action_name type + )const; /** * @brief Check authorizations of a vector of actions with provided keys, permission levels, and delay @@ -124,10 +126,10 @@ namespace eosio { namespace chain { void check_unlinkauth_authorization( const unlinkauth& unlink, const vector& auths )const; fc::microseconds check_canceldelay_authorization( const canceldelay& cancel, const vector& auths )const; - optional lookup_linked_permission( account_name authorizer_account, - scope_name code_account, - action_name type - )const; + std::optional lookup_linked_permission( account_name authorizer_account, + scope_name code_account, + action_name type + )const; }; } } /// namespace eosio::chain diff --git a/libraries/chain/include/eosio/chain/backing_store.hpp b/libraries/chain/include/eosio/chain/backing_store.hpp new file mode 100644 index 00000000000..b1fc9ae3738 --- /dev/null +++ b/libraries/chain/include/eosio/chain/backing_store.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include +#include + +namespace eosio { namespace chain { + + enum class backing_store_type { + CHAINBASE, // A name for regular users. Uses Chainbase. + ROCKSDB + }; + +}} // namespace eosio::chain + +namespace fc { +template <> +inline void to_variant(const eosio::chain::backing_store_type& store, fc::variant& v) { + v = (uint64_t)store; +} +template <> +inline void from_variant(const fc::variant& v, eosio::chain::backing_store_type& store) { + switch (store = (eosio::chain::backing_store_type)v.as_uint64()) { + case eosio::chain::backing_store_type::CHAINBASE: + case eosio::chain::backing_store_type::ROCKSDB: + return; + } + throw std::runtime_error("Invalid backing store name: " + v.as_string()); +} +} // namespace fc diff --git a/libraries/chain/include/eosio/chain/backing_store/chain_kv_payer.hpp b/libraries/chain/include/eosio/chain/backing_store/chain_kv_payer.hpp new file mode 100644 index 00000000000..aae8192c019 --- /dev/null +++ b/libraries/chain/include/eosio/chain/backing_store/chain_kv_payer.hpp @@ -0,0 +1,52 @@ +#pragma once + +#include +#include + +namespace eosio { namespace chain { namespace backing_store { + static constexpr auto payer_in_value_size = sizeof(account_name); + + inline static uint32_t actual_value_size(const uint32_t raw_value_size) { + EOS_ASSERT(raw_value_size >= payer_in_value_size, kv_rocksdb_bad_value_size_exception , "The size of value returned from RocksDB is less than payer's size"); + return (raw_value_size - payer_in_value_size); + } + + inline static account_name get_payer(const char* data) { + account_name payer; + memcpy(&payer, data, payer_in_value_size); + return payer; + } + + inline static const char* actual_value_start(const char* data) { + return data + payer_in_value_size; + } + + // used to create a payload of payer and char* buffer, or to take a char* buffer and extract the payload from it + // and a char* to the value portion of the payload. NOTE: this is meant to be a short + struct payer_payload { + payer_payload(const char* data, std::size_t size) + : value(actual_value_start(data)), value_size(actual_value_size(size)), payer(get_payer(data)) {} + + template + payer_payload(const CharCont& data) + : value(actual_value_start(data.data())), value_size(actual_value_size(data.size())), + payer(get_payer(data.data())) {} + + payer_payload(name payer, const char* val, std::size_t val_size) + : value(val), value_size(val_size), payer(payer) {} + + eosio::session::shared_bytes as_payload() const { + char payer_buf[payer_in_value_size]; + memcpy(payer_buf, &payer, payer_in_value_size); + return eosio::session::make_shared_bytes({std::string_view{payer_buf, + payer_in_value_size}, + std::string_view{value, value_size}}); + } + + // pointer to the actual value portion of the payload + const char* value; + // size of the actual value portion of the payload + std::size_t value_size; + account_name payer; + }; +}}} // ns eosio::chain::backing_store diff --git a/libraries/chain/include/eosio/chain/backing_store/db_chainbase_iter_store.hpp b/libraries/chain/include/eosio/chain/backing_store/db_chainbase_iter_store.hpp new file mode 100644 index 00000000000..48b9dcad474 --- /dev/null +++ b/libraries/chain/include/eosio/chain/backing_store/db_chainbase_iter_store.hpp @@ -0,0 +1,93 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace eosio { namespace chain { namespace backing_store { + +template +class db_chainbase_iter_store { + public: + db_chainbase_iter_store(){ + _end_iterator_to_table.reserve(8); + _iterator_to_object.reserve(32); + } + + /// Returns end iterator of the table. + int cache_table( const table_id_object& tobj ) { + auto itr = _table_cache.find(tobj.id); + if( itr != _table_cache.end() ) + return itr->second.second; + + auto ei = index_to_end_iterator(_end_iterator_to_table.size()); + _end_iterator_to_table.push_back( &tobj ); + _table_cache.emplace( tobj.id, make_pair(&tobj, ei) ); + return ei; + } + + const table_id_object& get_table( table_id_object::id_type i )const { + auto itr = _table_cache.find(i); + EOS_ASSERT( itr != _table_cache.end(), table_not_in_cache, "an invariant was broken, table should be in cache" ); + return *itr->second.first; + } + + int get_end_iterator_by_table_id( table_id_object::id_type i )const { + auto itr = _table_cache.find(i); + EOS_ASSERT( itr != _table_cache.end(), table_not_in_cache, "an invariant was broken, table should be in cache" ); + return itr->second.second; + } + + const table_id_object* find_table_by_end_iterator( int ei )const { + EOS_ASSERT( ei < -1, invalid_table_iterator, "not an end iterator" ); + auto indx = end_iterator_to_index(ei); + if( indx >= _end_iterator_to_table.size() ) return nullptr; + return _end_iterator_to_table[indx]; + } + + const T& get( int iterator ) { + EOS_ASSERT( iterator != -1, invalid_table_iterator, "invalid iterator" ); + EOS_ASSERT( iterator >= 0, table_operation_not_permitted, "dereference of end iterator" ); + EOS_ASSERT( (size_t)iterator < _iterator_to_object.size(), invalid_table_iterator, "iterator out of range" ); + auto result = _iterator_to_object[iterator]; + EOS_ASSERT( result, table_operation_not_permitted, "dereference of deleted object" ); + return *result; + } + + void remove( int iterator ) { + EOS_ASSERT( iterator != -1, invalid_table_iterator, "invalid iterator" ); + EOS_ASSERT( iterator >= 0, table_operation_not_permitted, "cannot call remove on end iterators" ); + EOS_ASSERT( (size_t)iterator < _iterator_to_object.size(), invalid_table_iterator, "iterator out of range" ); + + auto obj_ptr = _iterator_to_object[iterator]; + if( !obj_ptr ) return; + _iterator_to_object[iterator] = nullptr; + _object_to_iterator.erase( obj_ptr ); + } + + int add( const T& obj ) { + auto itr = _object_to_iterator.find( &obj ); + if( itr != _object_to_iterator.end() ) + return itr->second; + + _iterator_to_object.push_back( &obj ); + _object_to_iterator[&obj] = _iterator_to_object.size() - 1; + + return _iterator_to_object.size() - 1; + } + + private: + map> _table_cache; + vector _end_iterator_to_table; + vector _iterator_to_object; + map _object_to_iterator; + + /// Precondition: std::numeric_limits::min() < ei < -1 + /// Iterator of -1 is reserved for invalid iterators (i.e. when the appropriate table has not yet been created). + inline size_t end_iterator_to_index( int ei )const { return (-ei - 2); } + /// Precondition: indx < _end_iterator_to_table.size() <= std::numeric_limits::max() + inline int index_to_end_iterator( size_t indx )const { return -(indx + 2); } +}; /// class db_chainbase_iter_store + +}}} // namespace eosio::chain::backing_store diff --git a/libraries/chain/include/eosio/chain/backing_store/db_combined.hpp b/libraries/chain/include/eosio/chain/backing_store/db_combined.hpp new file mode 100644 index 00000000000..571bdca031d --- /dev/null +++ b/libraries/chain/include/eosio/chain/backing_store/db_combined.hpp @@ -0,0 +1,646 @@ +#pragma once +#include +#include +#include +#include +#include +#include + +namespace eosio { namespace chain { namespace backing_store { +using rocks_db_type = eosio::session::session; +using session_type = eosio::session::session; +using kv_undo_stack_ptr = std::unique_ptr>; +template +const char* contract_table_type(); + +// chainlib reserves prefixes 0x10 - 0x2F. +static constexpr char rocksdb_contract_kv_prefix = 0x11; // for KV API +static constexpr char rocksdb_contract_db_prefix = 0x12; // for DB API + +namespace detail { + class manage_stack { + public: + manage_stack(const kv_undo_stack_ptr& kv_undo_stack) : kv_undo_stack_(kv_undo_stack), undo_(kv_undo_stack->empty()) { + if (undo_) { + // Get a session to iterate over. + kv_undo_stack->push(); + } + } + + ~manage_stack(){ + if (undo_) { + kv_undo_stack_->undo(); + } + } + private: + const kv_undo_stack_ptr& kv_undo_stack_; + const bool undo_; + }; +} + +template +bool read_rocksdb_entry(const eosio::session::shared_bytes& actual_db_kv_key, + const eosio::session::shared_bytes& value, + F& function) { + uint64_t contract; + constexpr std::size_t type_size = 1; + std::size_t key_prefix_size = type_size + sizeof(contract); + EOS_ASSERT(actual_db_kv_key.size() >= key_prefix_size, database_exception, "Unexpected key in rocksdb"); + + auto key_buffer = std::vector{ actual_db_kv_key.data(), actual_db_kv_key.data() + actual_db_kv_key.size() }; + auto begin = std::begin(key_buffer) + type_size; + auto end = std::begin(key_buffer) + key_prefix_size; + b1::chain_kv::extract_key(begin, end, contract); + + const char* post_contract = actual_db_kv_key.data() + key_prefix_size; + const std::size_t remaining = actual_db_kv_key.size() - key_prefix_size; + return function(contract, post_contract, remaining, value.data(), value.size()); +}; + +struct table_id_object_view { + account_name code; //< code should not be changed within a chainbase modifier lambda + scope_name scope; //< scope should not be changed within a chainbase modifier lambda + table_name table; //< table should not be changed within a chainbase modifier lambda + account_name payer; + uint32_t count = 0; /// the number of elements in the table +}; + +struct blob { + std::string str_data; + std::size_t size() const { return str_data.size(); } + const char* data() const { return str_data.data(); } +}; + +template +inline DataStream &operator<<(DataStream &ds, const blob &b) { + fc::raw::pack(ds, b.str_data); + return ds; +} + +template +inline DataStream &operator>>(DataStream &ds, blob &b) { + fc::raw::unpack(ds, b.str_data); + return ds; +} + +struct primary_index_view { + static primary_index_view create(uint64_t key, const char* value, std::size_t value_size) { + backing_store::payer_payload pp(value, value_size); + return primary_index_view{key, pp.payer, {std::string{pp.value, pp.value + pp.value_size}}}; + } + uint64_t primary_key; + account_name payer; + blob value; +}; + +template +struct secondary_index_view { + uint64_t primary_key; + account_name payer; + SecondaryKeyType secondary_key; +}; + +template +class add_database_receiver { +public: + add_database_receiver(Receiver& receiver, const chainbase::database& db) : receiver_(receiver), db_(db) {} + template + void add_row(const Object& row) { + receiver_.add_row(row, db_); + } + + void add_row(fc::unsigned_int x){ + receiver_.add_row(x, db_); + } + +private: + Receiver& receiver_; + const chainbase::database& db_; +}; + +// used to wrap a rocksdb_contract_db_table_writer to collect all primary and secondary keys and report them after the +// table_id_object_view is reported. This also reports contract table related data in the format expected by snapshots +// (reporting the number of key rows (for primary and secondary key types) before reporting the rows. NOTE: using this +// collector really only makes sense with a key_context of complete, otherwise it is just extra processing with no +// benefit. It will produce invalid output if used with a key_context of complete_reverse +template +class rocksdb_whole_db_table_collector { +public: + rocksdb_whole_db_table_collector(Receiver &r) + : receiver_(r) {} + + void add_row(const chain::backing_store::primary_index_view& row){ + primary_indices_.emplace_back(row); + } + + void add_row(const chain::backing_store::secondary_index_view& row){ + secondary_indices().emplace_back(row); + } + + void add_row(const chain::backing_store::secondary_index_view& row){ + secondary_indices().emplace_back(row); + } + + void add_row(const chain::backing_store::secondary_index_view& row){ + secondary_indices().emplace_back(row); + } + + void add_row(const chain::backing_store::secondary_index_view& row){ + secondary_indices().emplace_back(row); + } + + void add_row(const chain::backing_store::secondary_index_view& row){ + secondary_indices().emplace_back(row); + } + + void add_row(const chain::backing_store::table_id_object_view& row){ + receiver_.add_row(row); + + auto add = [&rec=receiver_](const auto &row) { rec.add_row(row); }; + add(fc::unsigned_int(primary_indices_.size())); + // now that the whole table has been found, now we can report the individual keys + std::for_each(primary_indices_.begin(), primary_indices_.end(), add); + primary_indices_.clear(); + std::tuple secondary_key_types; + std::apply([this](auto... index) { (this->write_secondary_indices(index), ...); }, + secondary_key_types); + } +private: + template + auto& secondary_indices() { + return std::get>>(all_secondary_indices_); + } + + template + void write_secondary_indices(IndexType) { + auto& indices = secondary_indices(); + auto add = [this](const auto& row) { receiver_.add_row(row); }; + add(fc::unsigned_int(indices.size())); + std::for_each(indices.begin(), indices.end(), add); + indices.clear(); + } + + Receiver& receiver_; + std::vector primary_indices_; + template + using secondary_index_views = std::vector>; + std::tuple, + secondary_index_views, + secondary_index_views, + secondary_index_views, + secondary_index_views > + all_secondary_indices_; +}; + +constexpr auto process_all = []() { return true; }; + +// the context in which the table writer should be reporting to receiver_.add_row(...). NOTE: for any context indicating +// that it reports complete/valid table_id_object_views, for it to be valid it must be passed all keys belonging to that +// table, if not, the count portion of the table_id_object_view will not be accurate +enum class key_context { + complete, // report keys (via receiver_.add_row) as they are seen, so table will be after its keys (use rocksdb_whole_db_table_collector to reverse this) + complete_reverse, // report keys (via receiver_.add_row) as they are seen, used when reverse iterating through a table space (do not use with rocksdb_whole_db_table_collector) + standalone, // report an incomplete table (only code/scope/table valid) prior to reporting its keys + standalone_reverse, // report an incomplete table (only code/scope/table valid) prior to reporting its keys + table_only, // report only the table, primary and secondary keys are only processed enough to report a complete/valid table_id_object_view + table_only_reverse // report only the table, primary and secondary keys are only processed enough to report a complete/valid table_id_object_view +}; + +// processes any keys passed to it, reporting them to its receiver as it sees them. Use a key_context to adjust this behavior, +// pass in a keep_processing lambda to indicate to one of the walk_*** methods that processing should stop (like for +// limiting time querying the database for an PRC call) +template> +class rocksdb_contract_db_table_writer { +public: + using key_type = db_key_value_format::key_type; + + rocksdb_contract_db_table_writer(Receiver &r, key_context context, Function keep_processing = process_all) + : receiver_(r), context_(context), keep_processing_(keep_processing) {} + + explicit rocksdb_contract_db_table_writer(Receiver &r) + : receiver_(r), context_(key_context::complete), keep_processing_(process_all) {} + + void extract_primary_index(b1::chain_kv::bytes::const_iterator remaining, + b1::chain_kv::bytes::const_iterator key_end, const char *value, + std::size_t value_size) { + if (!is_table_only()) { + uint64_t primary_key; + EOS_ASSERT(b1::chain_kv::extract_key(remaining, key_end, primary_key), bad_composite_key_exception, + "DB intrinsic key-value store composite key is malformed"); + receiver_.add_row(primary_index_view::create(primary_key, value, value_size)); + } + ++primary_count_; + } + + template + void extract_secondary_index(b1::chain_kv::bytes::const_iterator remaining, + b1::chain_kv::bytes::const_iterator key_end, const char *value, + std::size_t value_size) { + if (!is_table_only()) { + IndexType secondary_key; + uint64_t primary_key; + EOS_ASSERT(b1::chain_kv::extract_key(remaining, key_end, secondary_key), bad_composite_key_exception, + "DB intrinsic key-value store composite key is malformed"); + EOS_ASSERT(b1::chain_kv::extract_key(remaining, key_end, primary_key), bad_composite_key_exception, + "DB intrinsic key-value store composite key is malformed"); + backing_store::payer_payload pp(value, value_size); + receiver_.add_row(secondary_index_view{primary_key, pp.payer, secondary_key}); + } + ++secondary_count(); + } + + // DB Intrinsic Keys + // PREFIX: + // db intrinsic keys are all prefaced with the following information: + // | contract | scope | table | type of key | + // so when iterating through the database, all keys for a given contract/scope/table will be processed together + // + // TYPE OF KEY: + // the "type of key" is an enum which ends up creating the following order for the type of keys: + // primary key + // primary to secondary key (not present in chainbase structure since it can have more than one index on the same structure -- these are ignored for this class's processing) + // uint64_t secondary key + // uint128_t secondary key + // key256_t secondary key + // float64_t secondary key + // float128_t secondary key + // table key (indicates end of table and payer) + // + // KEY PROCESSING (key_context): + // keys may be processed as: + // individual key (standalone) + // -- this can be any key - primary, secondary, or table + // table only (table_only, table_only_reversed) + // -- all keys for a table are passed in and processed only to report a valid table_id_object_view, if the table is + // passed in table_only, the table is determined to be complete when the table key is received, for + // table_only_reversed all of the table_id_object_view data, other than the count, is determined when the + // table key is received and the count is determined when the table is determined to be complete (either by a + // new key being received that is for a different table, or the complete() method is called externally). + // -- only the the table_id_object_view is reported, none of the other key's objects are reported + // complete (complete, complete_reverse) + // -- similar to table only, except all of the objects for the keys in the table are also reported + bool operator()(uint64_t contract, const char *key, std::size_t key_size, + const char *value, std::size_t value_size) { + b1::chain_kv::bytes composite_key(key, key + key_size); + auto[scope, table, remaining, type] = backing_store::db_key_value_format::get_prefix_thru_key_type( + composite_key); + + const name code = name{contract}; + if (!is_same_table(code, scope, table)) { + // since the table is being traversed in reverse order and + // we have identified a key for a new table, need to add + // the key count for the completed table and report it + complete(); + } + + if (!keep_processing_()) { + // performing the check after retrieving the prefix to provide similar results for RPC calls + // (indicating next scope) + stopped_processing_ = true; + table_context_ = table_id_object_view{code, scope, table, name{}, 0}; + return false; + } + + if (type == key_type::table) { + backing_store::payer_payload pp(value, value_size); + + + if (is_reversed()) { + // since table is reversed, we see the table key first, + // so we have the whole table context, except for the + // count of keys contained in the table, that will need + // to be added later once the next table is identified + // (so that we know that this table is + // complete - see complete()) + table_context_ = table_id_object_view{code, scope, table, pp.payer, 0}; + } + else { + table_context_ = table_id_object_view{code, scope, table, pp.payer, total_count()}; + receiver_.add_row(*table_context_); + } + + } else if (type != key_type::primary_to_sec) { + // for individual keys or reversed, need to report the table info for reference + check_context(code, scope, table); + std::invoke(extract_index_member_fun_[static_cast(type)], this, remaining, composite_key.end(), + value, value_size); + } + return true; + } + + std::optional stopped_at() const { + if (stopped_processing_) { + return table_context_; + } + return std::optional(); + } + + bool is_reversed() const { + return context_ == key_context::complete_reverse || context_ == key_context::table_only_reverse || context_ == key_context::standalone_reverse; + }; + + bool is_standalone() const { + return context_ == key_context::standalone || context_ == key_context::standalone_reverse; + }; + + bool is_table_only() const { + return context_ == key_context::table_only || context_ == key_context::table_only_reverse; + }; + + // called to indicate processing is complete (to allow completion of reversed table processing) + void complete() { + if (is_reversed() && table_context_ && !stopped_processing_) { + table_context_->count = total_count(); + receiver_.add_row(*table_context_); + } + } +private: + template + uint32_t& secondary_count() { + constexpr auto index = static_cast(db_key_value_format::derive_secondary_key_type()) - + static_cast(db_key_value_format::key_type::sec_i64); + return all_secondary_indices_count_[index]; + } + + void check_context(const name& code, const name& scope, const name& table) { + // this method only has to do with standalone processing + if (!is_standalone() || is_same_table(code, scope, table)) { + return; + } + table_context_ = table_id_object_view{code, scope, table, name{}, 0}; + receiver_.add_row(*table_context_); + } + + // calculates the total_count of keys, and zeros out the counts + uint32_t total_count() { + uint32_t total = primary_count_; + for (auto& count : all_secondary_indices_count_) { + total += count; + count = 0; + } + primary_count_ = 0; + return total; + } + + bool is_same_table(const name& code, const name& scope, const name& table) const { + return table_context_ && + table_context_->code == code && + table_context_->scope == scope && + table_context_->table == table; + } + + Receiver &receiver_; + const key_context context_; + Function keep_processing_; + std::optional table_context_; + + using extract_index_member_fun_t = void (rocksdb_contract_db_table_writer::*)( + b1::chain_kv::bytes::const_iterator, b1::chain_kv::bytes::const_iterator, const char *, std::size_t); + constexpr static unsigned member_fun_count = static_cast(key_type::sec_long_double) + 1; + const static extract_index_member_fun_t extract_index_member_fun_[member_fun_count]; + uint32_t primary_count_ = 0; + constexpr static unsigned secondary_indices = static_cast(db_key_value_format::key_type::sec_long_double) - + static_cast(db_key_value_format::key_type::sec_i64) + 1; + uint32_t all_secondary_indices_count_[secondary_indices] = {}; + + bool stopped_processing_ = false; +}; + +// array mapping db_key_value_format::key_type values (except table) to functions to break down to component parts +template +const typename rocksdb_contract_db_table_writer::extract_index_member_fun_t + rocksdb_contract_db_table_writer::extract_index_member_fun_[] = { + &rocksdb_contract_db_table_writer::extract_primary_index, + nullptr, // primary_to_sec type is covered by writing the secondary key type + &rocksdb_contract_db_table_writer::extract_secondary_index, + &rocksdb_contract_db_table_writer::extract_secondary_index, + &rocksdb_contract_db_table_writer::extract_secondary_index, + &rocksdb_contract_db_table_writer::extract_secondary_index, + &rocksdb_contract_db_table_writer::extract_secondary_index, +}; + +template> +class rocksdb_contract_kv_table_writer { +public: + rocksdb_contract_kv_table_writer(Receiver& r, const Function& keep_processing = process_all) + : receiver_(r), keep_processing_(keep_processing) {} + + bool operator()(uint64_t contract, const char* key, std::size_t key_size, + const char* value, std::size_t value_size) { + if (!keep_processing_()) { + return false; + } + // In KV RocksDB, payer and actual data are packed together. + // Extract them. + backing_store::payer_payload pp(value, value_size); + kv_object_view row{name(contract), + {{key, key + key_size}}, + {{pp.value, pp.value + pp.value_size}}, + pp.payer}; + receiver_.add_row(row); + return true; + } + +private: + Receiver& receiver_; + const Function& keep_processing_; +}; + +namespace detail { + // used to handle forward and reverse iteration and the limitations of no upper_bound + class iterator_pair { + public: + iterator_pair(const eosio::session::shared_bytes& begin_key, + const eosio::session::shared_bytes& end_key, + bool is_reverse, + kv_undo_stack_ptr::element_type::variant_type& session) : is_reverse_(is_reverse) { + EOS_ASSERT(begin_key < end_key, database_exception, "Invalid iterator_pair request: begin_key was greater than or equal to end_key."); + if (is_reverse_) { + current_ = session.lower_bound(end_key); + end_ = session.lower_bound(begin_key); + // since this is reverse iterating, then need to iterate backward if this is a greater-than iterator, + // to get a greater-than-or-equal reverse iterator + if (current_ == session.end() || (*current_).first > end_key) { + --current_; + EOS_ASSERT(current_ != session.end(), database_exception, "iterator_pair: failed to find lower bound of end_key"); + } + // since this is reverse iterating, then need to iterate backward to get a less-than iterator + --end_; + } + else { + current_ = session.lower_bound(begin_key); + end_ = session.lower_bound(end_key); + } + } + + bool valid() const { + return current_ != end_; + } + + void next() { + if (is_reverse_) + --current_; + else + ++current_; + } + + session_type::iterator::value_type get() const { + return *current_; + } + private: + const bool is_reverse_; + kv_undo_stack_ptr::element_type::variant_type::iterator current_; + kv_undo_stack_ptr::element_type::variant_type::iterator end_; + }; + + template + bool is_reversed(const rocksdb_contract_db_table_writer& writer) { + return writer.is_reversed(); + } + + template + bool is_reversed(const Function& ) { + return false; + } + + template + void complete(rocksdb_contract_db_table_writer& writer) { + writer.complete(); + } + + template + void complete(Function& ) { + } + +} + +template +void walk_rocksdb_entries_with_prefix(const kv_undo_stack_ptr& kv_undo_stack, + const eosio::session::shared_bytes& begin_key, + const eosio::session::shared_bytes& end_key, + F& function) { + auto session = kv_undo_stack->top(); + + detail::iterator_pair iter_pair(begin_key, end_key, detail::is_reversed(function), session); + bool keep_processing = true; + for (; keep_processing && iter_pair.valid(); iter_pair.next()) { + const auto data = iter_pair.get(); + // iterating through the session will always return a valid value + keep_processing = read_rocksdb_entry(data.first, *data.second, function); + } + // indicate processing is done + detail::complete(function); +}; + +// will walk through all entries with the given prefix, so if passed an exact key, it will match that key +// and any keys with that key as a prefix +template > +bool process_rocksdb_entry(Session& session, + const eosio::session::shared_bytes& key, + Receiver& receiver) { + if (!key) { + return false; + } + const auto value = session.read(key); + if (!value) { + return false; + } + const char prefix = key[0]; + if (prefix == rocksdb_contract_kv_prefix) { + rocksdb_contract_kv_table_writer kv_writer(receiver); + read_rocksdb_entry(key, *value, kv_writer); + } + else { + if (prefix != rocksdb_contract_db_prefix) { + char buffer[10]; + const auto len = sprintf(buffer, "%02x", static_cast(prefix)); + buffer[len] = '\0'; + FC_THROW_EXCEPTION(bad_composite_key_exception, + "Passed in key is prefixed with: ${prefix} which is neither the DB or KV prefix", + ("prefix", buffer)); + } + + rocksdb_contract_db_table_writer db_writer(receiver, key_context::standalone); + read_rocksdb_entry(key, *value, db_writer); + } + return true; +} + +template +const char* contract_table_type() { + if constexpr (std::is_same_v) { + return "kv_object_view"; + } + else if constexpr (std::is_same_v) { + return "table_id_object_view"; + } + else if constexpr (std::is_same_v) { + return "primary_index_view"; + } + else if constexpr (std::is_same_v>) { + return "secondary_index_view"; + } + else if constexpr (std::is_same_v>) { + return "secondary_index_view"; + } + else if constexpr (std::is_same_v>) { + return "secondary_index_view"; + } + else if constexpr (std::is_same_v>) { + return "secondary_index_view"; + } + else { + static_assert(std::is_same_v>); + return "secondary_index_view"; + } +} + +template +struct single_type_error_receiver { + template + void add_row(const Object& row) { + if constexpr (std::is_same_v) { + static_cast(this)->add_table_row(row); + } else if constexpr (std::is_same_v) { + static_cast(this)->add_only_row(row); + } else { + FC_THROW_EXCEPTION(Exception, "Invariant failure, should not receive an add_row call of type: ${type}", + ("type", contract_table_type>())); + } + } +}; + +template +struct table_only_error_receiver { + template + void add_row(const Object& row) { + if constexpr (std::is_same_v) { + static_cast(this)->add_table_row(row); + } else { + FC_THROW_EXCEPTION(Exception, "Invariant failure, should not receive an add_row call of type: ${type}", + ("type", contract_table_type>())); + } + } +}; + +}}} + +FC_REFLECT(eosio::chain::backing_store::table_id_object_view, (code)(scope)(table)(payer)(count) ) +FC_REFLECT(eosio::chain::backing_store::primary_index_view, (primary_key)(payer)(value) ) +REFLECT_SECONDARY(eosio::chain::backing_store::secondary_index_view) +REFLECT_SECONDARY(eosio::chain::backing_store::secondary_index_view) +REFLECT_SECONDARY(eosio::chain::backing_store::secondary_index_view) +REFLECT_SECONDARY(eosio::chain::backing_store::secondary_index_view) +REFLECT_SECONDARY(eosio::chain::backing_store::secondary_index_view) + +namespace fc { + inline + void to_variant( const eosio::chain::backing_store::blob& b, variant& v ) { + v = variant(base64_encode(b.str_data.data(), b.str_data.size())); + } + + inline + void from_variant( const variant& v, eosio::chain::backing_store::blob& b ) { + b.str_data = base64_decode(v.as_string()); + } +} diff --git a/libraries/chain/include/eosio/chain/backing_store/db_context.hpp b/libraries/chain/include/eosio/chain/backing_store/db_context.hpp new file mode 100644 index 00000000000..c171992fb8d --- /dev/null +++ b/libraries/chain/include/eosio/chain/backing_store/db_context.hpp @@ -0,0 +1,221 @@ +#pragma once + +#include +#include +#include + +namespace chainbase { + class database; +} + +namespace eosio { + namespace session { + struct rocksdb_t; + template + class session; + + template + class session_variant; + } +namespace chain { + + class apply_context; + +namespace backing_store { + struct db_context { + db_context(apply_context& c, const name& recv) : context(c), receiver(recv) {} + + virtual ~db_context() {} + + virtual int32_t db_store_i64(uint64_t scope, uint64_t table, account_name payer, uint64_t id, const char* buffer , size_t buffer_size) = 0; + + virtual void db_update_i64(int32_t itr, account_name payer, const char* buffer , size_t buffer_size) = 0; + + virtual void db_remove_i64(int32_t itr) = 0; + + virtual int32_t db_get_i64(int32_t itr, char* buffer , size_t buffer_size) = 0; + + virtual int32_t db_next_i64(int32_t itr, uint64_t& primary) = 0; + + virtual int32_t db_previous_i64(int32_t itr, uint64_t& primary) = 0; + + virtual int32_t db_find_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) = 0; + + virtual int32_t db_lowerbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) = 0; + + virtual int32_t db_upperbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) = 0; + + virtual int32_t db_end_i64(uint64_t code, uint64_t scope, uint64_t table) = 0; + + /** + * interface for uint64_t secondary + */ + virtual int32_t db_idx64_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, + const uint64_t& secondary) = 0; + + virtual void db_idx64_update(int32_t iterator, account_name payer, const uint64_t& secondary) = 0; + + virtual void db_idx64_remove(int32_t iterator) = 0; + + virtual int32_t db_idx64_find_secondary(uint64_t code, uint64_t scope, uint64_t table, + const uint64_t& secondary, uint64_t& primary) = 0; + + virtual int32_t db_idx64_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, + uint64_t primary) = 0; + + virtual int32_t db_idx64_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, + uint64_t& primary) = 0; + + virtual int32_t db_idx64_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, + uint64_t& primary) = 0; + + virtual int32_t db_idx64_end(uint64_t code, uint64_t scope, uint64_t table) = 0; + + virtual int32_t db_idx64_next(int32_t iterator, uint64_t& primary) = 0; + + virtual int32_t db_idx64_previous(int32_t iterator, uint64_t& primary) = 0; + + /** + * interface for uint128_t secondary + */ + virtual int32_t db_idx128_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, + const uint128_t& secondary) = 0; + + virtual void db_idx128_update(int32_t iterator, account_name payer, const uint128_t& secondary) = 0; + + virtual void db_idx128_remove(int32_t iterator) = 0; + + virtual int32_t db_idx128_find_secondary(uint64_t code, uint64_t scope, uint64_t table, + const uint128_t& secondary, uint64_t& primary) = 0; + + virtual int32_t db_idx128_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, + uint64_t primary) = 0; + + virtual int32_t db_idx128_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, + uint64_t& primary) = 0; + + virtual int32_t db_idx128_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, + uint64_t& primary) = 0; + + virtual int32_t db_idx128_end(uint64_t code, uint64_t scope, uint64_t table) = 0; + + virtual int32_t db_idx128_next(int32_t iterator, uint64_t& primary) = 0; + + virtual int32_t db_idx128_previous(int32_t iterator, uint64_t& primary) = 0; + + /** + * interface for 256-bit interger secondary + */ + virtual int32_t db_idx256_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, + const uint128_t* data) = 0; + + virtual void db_idx256_update(int32_t iterator, account_name payer, const uint128_t* data) = 0; + + virtual void db_idx256_remove(int32_t iterator) = 0; + + virtual int32_t db_idx256_find_secondary(uint64_t code, uint64_t scope, uint64_t table, const uint128_t* data, + uint64_t& primary) = 0; + + virtual int32_t db_idx256_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, + uint64_t primary) = 0; + + virtual int32_t db_idx256_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, + uint64_t& primary) = 0; + + virtual int32_t db_idx256_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, + uint64_t& primary) = 0; + + virtual int32_t db_idx256_end(uint64_t code, uint64_t scope, uint64_t table) = 0; + + virtual int32_t db_idx256_next(int32_t iterator, uint64_t& primary) = 0; + + virtual int32_t db_idx256_previous(int32_t iterator, uint64_t& primary) = 0; + + /** + * interface for double secondary + */ + virtual int32_t db_idx_double_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, + const float64_t& secondary) = 0; + + virtual void db_idx_double_update(int32_t iterator, account_name payer, const float64_t& secondary) = 0; + + virtual void db_idx_double_remove(int32_t iterator) = 0; + + virtual int32_t db_idx_double_find_secondary(uint64_t code, uint64_t scope, uint64_t table, + const float64_t& secondary, uint64_t& primary) = 0; + + virtual int32_t db_idx_double_find_primary(uint64_t code, uint64_t scope, uint64_t table, + float64_t& secondary, uint64_t primary) = 0; + + virtual int32_t db_idx_double_lowerbound(uint64_t code, uint64_t scope, uint64_t table, float64_t& secondary, + uint64_t& primary) = 0; + + virtual int32_t db_idx_double_upperbound(uint64_t code, uint64_t scope, uint64_t table, float64_t& secondary, + uint64_t& primary) = 0; + + virtual int32_t db_idx_double_end(uint64_t code, uint64_t scope, uint64_t table) = 0; + + virtual int32_t db_idx_double_next(int32_t iterator, uint64_t& primary) = 0; + + virtual int32_t db_idx_double_previous(int32_t iterator, uint64_t& primary) = 0; + + /** + * interface for long double secondary + */ + virtual int32_t db_idx_long_double_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, + const float128_t& secondary) = 0; + + virtual void db_idx_long_double_update(int32_t iterator, account_name payer, const float128_t& secondary) = 0; + + virtual void db_idx_long_double_remove(int32_t iterator) = 0; + + virtual int32_t db_idx_long_double_find_secondary(uint64_t code, uint64_t scope, uint64_t table, + const float128_t& secondary, uint64_t& primary) = 0; + + virtual int32_t db_idx_long_double_find_primary(uint64_t code, uint64_t scope, uint64_t table, + float128_t& secondary, uint64_t primary) = 0; + + virtual int32_t db_idx_long_double_lowerbound(uint64_t code, uint64_t scope, uint64_t table, + float128_t& secondary, uint64_t& primary) = 0; + + virtual int32_t db_idx_long_double_upperbound(uint64_t code, uint64_t scope, uint64_t table, + float128_t& secondary, uint64_t& primary) = 0; + + virtual int32_t db_idx_long_double_end(uint64_t code, uint64_t scope, uint64_t table) = 0; + + virtual int32_t db_idx_long_double_next(int32_t iterator, uint64_t& primary) = 0; + + virtual int32_t db_idx_long_double_previous(int32_t iterator, uint64_t& primary) = 0; + + static std::string table_event(name code, name scope, name table); + static std::string table_event(name code, name scope, name table, name qualifier); + static void log_insert_table(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, account_name payer); + static void log_remove_table(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, account_name payer); + static void log_row_insert(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, + account_name payer, account_name primkey, const char* buffer, size_t buffer_size); + static void log_row_update(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, + account_name old_payer, account_name new_payer, account_name primkey, + const char* old_buffer, size_t old_buffer_size, const char* new_buffer, size_t new_buffer_size); + static void log_row_remove(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, + account_name payer, account_name primkey, const char* buffer, size_t buffer_size); + static storage_usage_trace add_table_trace(uint32_t action_id, std::string&& event_id); + static storage_usage_trace rem_table_trace(uint32_t action_id, std::string&& event_id); + static storage_usage_trace row_add_trace(uint32_t action_id, std::string&& event_id); + static storage_usage_trace row_update_trace(uint32_t action_id, std::string&& event_id); + static storage_usage_trace row_update_add_trace(uint32_t action_id, std::string&& event_id); + static storage_usage_trace row_update_rem_trace(uint32_t action_id, std::string&& event_id); + static storage_usage_trace row_rem_trace(uint32_t action_id, std::string&& event_id); + static storage_usage_trace secondary_add_trace(uint32_t action_id, std::string&& event_id); + static storage_usage_trace secondary_rem_trace(uint32_t action_id, std::string&& event_id); + static storage_usage_trace secondary_update_add_trace(uint32_t action_id, std::string&& event_id); + static storage_usage_trace secondary_update_rem_trace(uint32_t action_id, std::string&& event_id); + void update_db_usage(const account_name& payer, int64_t delta, const storage_usage_trace& trace); + apply_context& context; + name receiver; + }; + + std::unique_ptr create_db_chainbase_context(apply_context& context, name receiver); + std::unique_ptr create_db_rocksdb_context(apply_context& context, name receiver, + eosio::session::session_variant, eosio::session::session>> session); + +}}} // ns eosio::chain::backing_store diff --git a/libraries/chain/include/eosio/chain/backing_store/db_key_value_any_lookup.hpp b/libraries/chain/include/eosio/chain/backing_store/db_key_value_any_lookup.hpp new file mode 100644 index 00000000000..56dbf177cba --- /dev/null +++ b/libraries/chain/include/eosio/chain/backing_store/db_key_value_any_lookup.hpp @@ -0,0 +1,78 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace eosio { namespace chain { namespace backing_store { + struct db_context; + using end_of_prefix = db_key_value_format::end_of_prefix; + + struct key_bundle { + key_bundle(const b1::chain_kv::bytes& ck, name code); + + eosio::session::shared_bytes full_key; + }; + + struct prefix_bundle { + prefix_bundle(const b1::chain_kv::bytes& composite_key, end_of_prefix prefix_end, name code); + + eosio::session::shared_bytes full_key; + eosio::session::shared_bytes prefix_key; + }; + + template + struct value_bundle : Key { + value_bundle(Key&& key, std::optional&& v) : Key(std::move(key)), value(std::move(v)) {} + value_bundle(const Key& key, std::optional&& v) : Key(key), value(std::move(v)) {} + std::optional value; + }; + + using kv_bundle = value_bundle; + using pv_bundle = value_bundle; + + struct db_key_value_any_lookup { + using session_type = eosio::session::session>; + using session_variant_type = eosio::session::session_variant; + using shared_bytes = eosio::session::shared_bytes; + + db_key_value_any_lookup(db_context& c, session_variant_type session) : parent(c), current_session(session) {} + + static key_bundle get_slice(name code, name scope, name table); + static key_bundle get_table_end_slice(name code, name scope, name table); + void add_table_if_needed(const shared_bytes& key, account_name payer); + void remove_table_if_empty(const shared_bytes& key); + template + int32_t get_end_iter(name code, name scope, name table, IterStore& iter_store) { + const auto table_key = get_table_end_slice(code, scope, table); + auto value = current_session.read(table_key.full_key); + if (!value) { + return iter_store.invalid_iterator(); + } + + const unique_table t { code, scope, table }; + const auto table_ei = iter_store.cache_table(t); + return table_ei; + } + + bool match_prefix(const shared_bytes& shorter, const shared_bytes& longer); + bool match_prefix(const shared_bytes& shorter, const session_variant_type::iterator& iter); + + bool match(const shared_bytes& lhs, const shared_bytes& rhs); + bool match(const shared_bytes& lhs, const session_variant_type::iterator& iter); + + db_context& parent; + session_variant_type current_session; + static constexpr int64_t table_overhead = config::billable_size_v; + static constexpr int64_t overhead = config::billable_size_v; + // this is used for any value that just needs something in it to distinguish it from the invalid value + static const shared_bytes useless_value; + }; + +}}} // ns eosio::chain::backing_store diff --git a/libraries/chain/include/eosio/chain/backing_store/db_key_value_format.hpp b/libraries/chain/include/eosio/chain/backing_store/db_key_value_format.hpp new file mode 100644 index 00000000000..0edcbc143c7 --- /dev/null +++ b/libraries/chain/include/eosio/chain/backing_store/db_key_value_format.hpp @@ -0,0 +1,581 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace eosio { + namespace session { + class shared_bytes; + } + +namespace chain { namespace backing_store { namespace db_key_value_format { + using key256_t = std::array; + + // NOTE: very limited use till redesign + constexpr uint64_t db_type_and_code_size = sizeof(char) + sizeof(name); // 1 (db type) + 8 (contract) + + b1::chain_kv::bytes create_primary_key(name scope, name table, uint64_t primary_key); + + bool get_primary_key(const b1::chain_kv::bytes& composite_key, name& scope, name& table, uint64_t& primary_key); + + // ordering affects key ordering + enum class key_type : char { + primary = 0, + primary_to_sec = 1, // used to lookup the secondary keys for a specific primary key + sec_i64 = 2, + sec_i128 = 3, + sec_i256 = 4, + sec_double = 5, + sec_long_double = 6, // changes needed in determine_sec_type to accommodate new types + table = std::numeric_limits::max() // require to be highest value for a given scope/table + }; + + using intermittent_decomposed_values = std::tuple; + + namespace detail { + + template + struct determine_sec_type {}; + + template<> + struct determine_sec_type { static constexpr key_type kt = key_type::sec_i64; }; + + template<> + struct determine_sec_type { static constexpr key_type kt = key_type::sec_i128; }; + + template<> + struct determine_sec_type { static constexpr key_type kt = key_type::sec_i256; }; + + template<> + struct determine_sec_type { static constexpr key_type kt = key_type::sec_double; }; + + template<> + struct determine_sec_type { static constexpr key_type kt = key_type::sec_long_double; }; + + template + struct associated; + + template<> + struct associated { + using stored_key = eosio::session::shared_bytes; + }; + + template + struct associated { + using stored_key = b1::chain_kv::bytes; + }; + + template + constexpr std::size_t prefix_size() { + constexpr uint64_t legacy_prefix_size = sizeof(name) * 2; // 8 (scope) + 8 (table) + if constexpr (std::is_same_v) { + constexpr uint64_t db_type_and_code_size = sizeof(char) + sizeof(name); // 1 (db type) + 4 (contract) + return db_type_and_code_size + legacy_prefix_size; + } + else if constexpr (std::is_same_v) { + return legacy_prefix_size; + } + else { + static_assert(std::is_same_v); + return legacy_prefix_size; + } + } + + template + constexpr void consistent_keys() { + if constexpr (std::is_same_v) { + return; + } + else { + static_assert(!std::is_same_v); + static_assert(!std::is_same_v); + } + } + + constexpr std::size_t key_size(key_type kt); + + std::string to_string(const key_type& kt); + + b1::chain_kv::bytes prepare_composite_key_prefix(name scope, name table, std::size_t type_size, std::size_t key_size, std::size_t extension_size); + + b1::chain_kv::bytes prepare_composite_key(name scope, name table, std::size_t key_size, key_type kt, std::size_t extension_size); + + using intermittent_decomposed_prefix_values = std::tuple; + + intermittent_decomposed_prefix_values extract_from_composite_key_prefix(b1::chain_kv::bytes::const_iterator begin, b1::chain_kv::bytes::const_iterator key_end); + + intermittent_decomposed_values extract_from_composite_key(b1::chain_kv::bytes::const_iterator begin, b1::chain_kv::bytes::const_iterator key_end); + + template + void append_identifier_and_key(b1::chain_kv::bytes& composite_key, const Key& some_key, key_type kt = detail::determine_sec_type::kt) { + composite_key.push_back(static_cast(kt)); + b1::chain_kv::append_key(composite_key, some_key); + } + + template + void append_primary_and_secondary_key_types(b1::chain_kv::bytes& composite_key) { + composite_key.push_back(static_cast(key_type::primary_to_sec)); + // NOTE: db_key_value_any_lookup end_of_prefix::at_prim_to_sec_type tied to this layout + const key_type kt = determine_sec_type::kt; + composite_key.push_back(static_cast(kt)); + } + + template + void append_primary_and_secondary_keys(b1::chain_kv::bytes& composite_key, uint64_t primary_key, const Key& sec_key) { + append_primary_and_secondary_key_types(composite_key); + // NOTE: db_key_value_any_lookup end_of_prefix::at_prim_to_sec_primary_key tied to this layout + b1::chain_kv::append_key(composite_key, primary_key); + b1::chain_kv::append_key(composite_key, sec_key); + } + + template + bool validate_primary_to_sec_type_prefix(const CharKey& type_prefix, std::size_t start_prefix_size, key_type expected_sec_kt) { + // possibly add the extra_prefix_size (to cover db_type and contract for a full key) + const auto prefix_minimum_size = start_prefix_size + sizeof(key_type); + const auto both_types_size = prefix_minimum_size + sizeof(key_type); + const auto both_types_and_primary_size = both_types_size + sizeof(uint64_t); + const bool has_sec_type = type_prefix.size() >= both_types_size; + EOS_ASSERT(type_prefix.size() == prefix_minimum_size || type_prefix.size() == both_types_size || + type_prefix.size() == both_types_and_primary_size, bad_composite_key_exception, + "DB intrinsic key-value validate_primary_to_sec_type_prefix was passed a prefix key size: ${s1} " + "bytes that was not either: ${s2}, ${s3}, or ${s4} bytes long", + ("s1", type_prefix.size())("s2", prefix_minimum_size)("s3", both_types_and_primary_size) + ("s4", both_types_and_primary_size)); + const char* prefix_ptr = type_prefix.data() + start_prefix_size; + const key_type prefix_type_kt {*(prefix_ptr++)}; + EOS_ASSERT(key_type::primary_to_sec == prefix_type_kt, bad_composite_key_exception, + "DB intrinsic key-value validate_primary_to_sec_type_prefix was passed a prefix that was the wrong " + "type: ${type}, it should have been of type: ${type2}", + ("type", to_string(prefix_type_kt))("type2", to_string(key_type::primary_to_sec))); + if (has_sec_type) { + const key_type prefix_sec_kt {*(prefix_ptr)}; + EOS_ASSERT(expected_sec_kt == prefix_sec_kt, bad_composite_key_exception, + "DB intrinsic key-value validate_primary_to_sec_type_prefix was passed a prefix that was the " + "wrong secondary type: ${type}, it should have been of type: ${type2}", + ("type", to_string(prefix_sec_kt))("type2", to_string(expected_sec_kt))); + } + return has_sec_type; + } + + template + void validate_primary_to_sec_type_size(const CharKey& full_key, std::size_t start_prefix_size) { + const auto prefix_minimum_size = start_prefix_size + sizeof(key_type); + const auto both_types_size = prefix_minimum_size + sizeof(key_type); + const auto full_size = both_types_size + sizeof(uint64_t) + sizeof(Key); + EOS_ASSERT(full_key.size() == full_size, bad_composite_key_exception, + "DB intrinsic key-value validate_primary_to_sec_type_size was passed a full key size: ${s1} bytes that was not at least " + "larger than the size that would include the secondary type: ${s2}", + ("s1", full_key.size())("s2", full_size)); + } + + template + bool verify_primary_to_sec_type(const CharKey1& key, const CharKey2& type_prefix) { + detail::consistent_keys(); + + const std::size_t start_prefix_size = prefix_size(); + // >> validate that the type prefix is actually of primary_to_sec key type + const key_type expected_sec_kt = determine_sec_type::kt; + const auto check_sec_type = !validate_primary_to_sec_type_prefix(type_prefix, start_prefix_size, expected_sec_kt); + + // check if key is prefixed with type_prefix + if (memcmp(type_prefix.data(), key.data(), type_prefix.size()) != 0) { + return false; + } + if (check_sec_type) { + const key_type prefix_sec_kt {*(key.data() + start_prefix_size + sizeof(key_type))}; + // key was not the right sec type + if (expected_sec_kt != prefix_sec_kt) { + return false; + } + } + + validate_primary_to_sec_type_size(key, start_prefix_size); + return true; + } + + template + std::pair locate_trailing_primary_and_secondary_keys(const CharKey& key) { + // this method just points to the data, it expects validation to be done prior to calling + const auto both_type_size = prefix_size() + sizeof(key_type) + sizeof(key_type); + const auto primary_offset = key.data() + both_type_size; + return { rocksdb::Slice{primary_offset, sizeof(uint64_t)}, + rocksdb::Slice{primary_offset + sizeof(uint64_t), sizeof(Key)}}; + } + + template + void extract_trailing_primary_and_secondary_keys(const CharKey& key, uint64_t& primary_key, Key& sec_key) { + const auto key_offsets = locate_trailing_primary_and_secondary_keys(key); + const auto keys_sizes = key_offsets.first.size() + key_offsets.second.size(); + const auto start_offset = key_offsets.first.data(); + const b1::chain_kv::bytes composite_trailing_prim_sec_key {start_offset, start_offset + keys_sizes}; + auto composite_loc = composite_trailing_prim_sec_key.cbegin(); + EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_trailing_prim_sec_key.cend(), primary_key), bad_composite_key_exception, + "DB intrinsic key-value store invariant has changed, extract_key should only fail if the remaining " + "string size is less than the sizeof(uint64_t)"); + EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_trailing_prim_sec_key.cend(), sec_key), bad_composite_key_exception, + "DB intrinsic key-value store invariant has changed, extract_key should only fail if the remaining " + "string size is less than: ${s} bytes, which was already verified", ("s", key_offsets.second.size())); + EOS_ASSERT(composite_loc == composite_trailing_prim_sec_key.cend(), bad_composite_key_exception, + "DB intrinsic key-value store invariant has changed, calls to extract the pimary and secondary " + "keys should have resulted in the passed in buffer being consumed fully"); + } + + template + auto assemble(std::array&& data) { + if constexpr (std::is_same_v) { + return session::make_shared_bytes(std::move(data)); + } + else { + b1::chain_kv::bytes key_data; + const std::size_t init = 0; + const std::size_t length = std::accumulate(data.begin(), data.end(), init, + [](std::size_t a, const rocksdb::Slice& b) { return a + b.size(); }); + key_data.reserve(length); + for (const auto& d : data) { + key_data.insert(key_data.end(), d.data(), d.data() + d.size()); + } + return key_data; + } + } + } + + template + constexpr key_type derive_secondary_key_type() { + return detail::determine_sec_type::kt; + } + + template + b1::chain_kv::bytes create_secondary_key(name scope, name table, const Key& sec_key, uint64_t primary_key) { + const key_type kt = detail::determine_sec_type::kt; + b1::chain_kv::bytes composite_key = detail::prepare_composite_key(scope, table, sizeof(Key), kt, sizeof(primary_key)); + b1::chain_kv::append_key(composite_key, sec_key); + b1::chain_kv::append_key(composite_key, primary_key); + return composite_key; + } + + b1::chain_kv::bytes create_prefix_type_key(name scope, name table, key_type kt); + + template + b1::chain_kv::bytes create_prefix_secondary_key(name scope, name table, const Key& sec_key) { + const key_type kt = detail::determine_sec_type::kt; + b1::chain_kv::bytes composite_key = detail::prepare_composite_key(scope, table, sizeof(Key), kt, 0); + b1::chain_kv::append_key(composite_key, sec_key); + return composite_key; + } + + template + bool get_secondary_key(const b1::chain_kv::bytes& composite_key, name& scope, name& table, Key& sec_key, uint64_t& primary_key) { + b1::chain_kv::bytes::const_iterator composite_loc; + key_type kt = key_type::sec_i64; + std::tie(scope, table, composite_loc, kt) = detail::extract_from_composite_key(composite_key.cbegin(), composite_key.cend()); + constexpr static auto expected_kt = detail::determine_sec_type::kt; + if (kt != expected_kt) { + return false; + } + EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_key.cend(), sec_key), bad_composite_key_exception, + "DB intrinsic key-value store composite key is malformed, it is supposed to have an underlying: " + + detail::to_string(kt)); + EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_key.cend(), primary_key), bad_composite_key_exception, + "DB intrinsic key-value store composite key is malformed, it is supposed to have a trailing primary key"); + return true; + } + + struct secondary_key_pair { + b1::chain_kv::bytes secondary_key; + b1::chain_kv::bytes primary_to_secondary_key; + }; + + template + secondary_key_pair create_secondary_key_pair(name scope, name table, const Key& sec_key, uint64_t primary_key) { + b1::chain_kv::bytes secondary_key = create_secondary_key(scope, table, sec_key, primary_key); + b1::chain_kv::bytes primary_to_secondary_key(secondary_key.begin(), secondary_key.begin() + detail::prefix_size()); + detail::append_primary_and_secondary_keys(primary_to_secondary_key, primary_key, sec_key); + return { std::move(secondary_key), std::move(primary_to_secondary_key) }; + } + + template + CharKey create_secondary_key_from_primary_to_sec_key(const CharKey& primary_to_sec_key) { + // checking size first, to validate that the key is the correct size before it is split + const auto start_prefix_size = detail::prefix_size(); + detail::validate_primary_to_sec_type_size(primary_to_sec_key, start_prefix_size); + const auto both_types_and_primary_size = detail::prefix_size() + sizeof(key_type) + sizeof(key_type) + sizeof(uint64_t); + const rocksdb::Slice prefix {primary_to_sec_key.data(), both_types_and_primary_size}; + // now validating the front end of the key + const key_type expected_sec_kt = detail::determine_sec_type::kt; + // NOTE: using Slice, but it has the new db type and contract prefix + const auto sec_kt_checked = detail::validate_primary_to_sec_type_prefix(prefix, start_prefix_size, expected_sec_kt); + EOS_ASSERT(sec_kt_checked, bad_composite_key_exception, + "DB intrinsic invariant failure, key-value create_secondary_key_from_primary_to_sec_key was passed a " + "key that was verified as long enough but validate_primary_to_sec_type_prefix indicated that it wasn't " + "long enough to verify the secondary type"); + const auto key_offsets = detail::locate_trailing_primary_and_secondary_keys(primary_to_sec_key); + using return_type = typename detail::associated::stored_key; + const auto new_kt = static_cast(expected_sec_kt); + const char* new_kt_ptr = &new_kt; + // already have everything in char format/order, so just need to assemble it + return detail::assemble<4, return_type>({rocksdb::Slice{primary_to_sec_key.data(), start_prefix_size}, + rocksdb::Slice{new_kt_ptr, 1}, + key_offsets.second, + key_offsets.first}); + } + + template + bool get_trailing_sec_prim_keys(const CharKey1& full_key, const CharKey2& secondary_type_prefix, Key& sec_key, uint64_t& primary_key) { + detail::consistent_keys(); + + // >> verify that secondary_type_prefix is really a valid secondary type prefix + + // possibly add the extra_prefix_size (to cover db_type and contract for a full key) + const auto sec_type_prefix_size = secondary_type_prefix.size(); + const auto prefix_type_size = detail::prefix_size() + sizeof(key_type); + EOS_ASSERT(sec_type_prefix_size == prefix_type_size, bad_composite_key_exception, + "DB intrinsic key-value get_trailing_sec_prim_keys was passed a secondary_type_prefix with key size: ${s1} bytes " + "which is not equal to the expected size: ${s2}", ("s1", sec_type_prefix_size)("s2", prefix_type_size)); + const key_type expected_kt = detail::determine_sec_type::kt; + const key_type actual_kt {*(secondary_type_prefix.data() + sec_type_prefix_size - 1)}; + EOS_ASSERT(expected_kt == actual_kt, bad_composite_key_exception, + "DB intrinsic key-value get_trailing_sec_prim_keys was passed a key that was the wrong type: ${type}," + " it should have been of type: ${type2}", + ("type", detail::to_string(actual_kt))("type2", detail::to_string(expected_kt))); + const auto keys_size = sizeof(sec_key) + sizeof(primary_key); + EOS_ASSERT(full_key.size() == sec_type_prefix_size + keys_size, bad_composite_key_exception, + "DB intrinsic key-value get_trailing_sec_prim_keys was passed a full key size: ${s1} bytes that was not exactly " + "${s2} bytes (the size of the secondary key and a primary key) larger than the secondary_type_prefix " + "size: ${s3}", ("s1", full_key.size())("s2", keys_size)("s3", sec_type_prefix_size)); + + // >> identify if the passed in key matches the prefix + + const auto comp = memcmp(secondary_type_prefix.data(), full_key.data(), sec_type_prefix_size); + if (comp != 0) { + return false; + } + auto start_offset = full_key.data() + sec_type_prefix_size; + const b1::chain_kv::bytes composite_trailing_sec_prim_key {start_offset, start_offset + keys_size}; + auto composite_loc = composite_trailing_sec_prim_key.cbegin(); + EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_trailing_sec_prim_key.cend(), sec_key), bad_composite_key_exception, + "DB intrinsic key-value store invariant has changed, extract_key should only fail if the remaining " + "string size is less than: ${s} bytes, which was already verified", ("s", keys_size)); + EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_trailing_sec_prim_key.cend(), primary_key), bad_composite_key_exception, + "DB intrinsic key-value store invariant has changed, extract_key should only fail if the remaining " + "string size is less than the sizeof(uint64_t)"); + return true; + } + + bool get_trailing_primary_key(const rocksdb::Slice& full_key, const rocksdb::Slice& secondary_key_prefix, uint64_t& primary_key); + + template + b1::chain_kv::bytes create_primary_to_secondary_key(name scope, name table, uint64_t primary_key, const Key& sec_key) { + constexpr static auto type_size = sizeof(key_type); + auto composite_key = detail::prepare_composite_key_prefix(scope, table, type_size * 2, sizeof(primary_key), sizeof(sec_key)); + detail::append_primary_and_secondary_keys(composite_key, primary_key, sec_key); + return composite_key; + } + + template + b1::chain_kv::bytes create_prefix_primary_to_secondary_key(name scope, name table, uint64_t primary_key) { + constexpr static auto type_size = sizeof(key_type); + constexpr static auto zero_size = 0; + auto composite_key = detail::prepare_composite_key_prefix(scope, table, type_size * 2, sizeof(primary_key), zero_size); + detail::append_primary_and_secondary_key_types(composite_key); + b1::chain_kv::append_key(composite_key, primary_key); + return composite_key; + } + + template + b1::chain_kv::bytes create_prefix_primary_to_secondary_key(name scope, name table) { + constexpr static auto type_size = sizeof(key_type); + constexpr static auto zero_size = 0; + auto composite_key = detail::prepare_composite_key_prefix(scope, table, type_size * 2, zero_size, zero_size); + detail::append_primary_and_secondary_key_types(composite_key); + return composite_key; + } + + enum class prim_to_sec_type_result { invalid_type, wrong_sec_type, valid_type }; + template + prim_to_sec_type_result get_primary_to_secondary_keys(const b1::chain_kv::bytes& composite_key, name& scope, name& table, uint64_t& primary_key, Key& sec_key) { + b1::chain_kv::bytes::const_iterator composite_loc; + key_type kt = key_type::sec_i64; + std::tie(scope, table, composite_loc, kt) = detail::extract_from_composite_key(composite_key.cbegin(), composite_key.cend()); + if (kt != key_type::primary_to_sec) { + return prim_to_sec_type_result::invalid_type; + } + EOS_ASSERT(composite_loc != composite_key.cend(), bad_composite_key_exception, + "DB intrinsic key-value store composite key is malformed, it does not contain an indication of the " + "type of the secondary-key"); + const key_type sec_kt{*composite_loc++}; + constexpr static auto expected_kt = detail::determine_sec_type::kt; + if (sec_kt != expected_kt) { + return prim_to_sec_type_result::wrong_sec_type; + } + EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_key.cend(), primary_key), bad_composite_key_exception, + "DB intrinsic key-value store composite key is malformed, it is supposed to have a trailing primary key"); + + EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_key.cend(), sec_key), bad_composite_key_exception, + "DB intrinsic key-value store composite key is malformed, it is supposed to have an underlying: " + + detail::to_string(sec_kt)); + return prim_to_sec_type_result::valid_type; + } + + template + bool get_trailing_prim_sec_keys(const CharKey1& full_key, const CharKey2& primary_to_sec_type_prefix, + uint64_t& primary_key, Key& sec_key) { + detail::consistent_keys(); + + // >> verify that secondary_type_prefix is really a valid secondary type prefix + + // possibly add the extra_prefix_size (to cover db_type and contract for a full key) + if (!detail::verify_primary_to_sec_type(full_key, primary_to_sec_type_prefix)) { + return false; + } + + detail::extract_trailing_primary_and_secondary_keys(full_key, primary_key, sec_key); + return true; + } + + template + bool get_trailing_secondary_key(const CharKey1& full_key, const CharKey2& sec_type_trailing_prefix, Key& sec_key) { + detail::consistent_keys(); + // possibly add the extra_prefix_size (to cover db_type and contract for a full key) + const auto sec_type_trailing_prefix_size = sec_type_trailing_prefix.size(); + const auto prefix_with_primary_key_size = detail::prefix_size() + sizeof(key_type) * 2 + sizeof(uint64_t); + EOS_ASSERT(sec_type_trailing_prefix_size == prefix_with_primary_key_size, bad_composite_key_exception, + "DB intrinsic key-value get_trailing_secondary_key was passed a sec_type_trailing_prefix with key size: ${s1} bytes " + "which is not equal to the expected size: ${s2}", ("s1", sec_type_trailing_prefix_size)("s2", prefix_with_primary_key_size)); + if (!detail::verify_primary_to_sec_type(full_key, sec_type_trailing_prefix)) { + return false; + } + const auto key_size = sizeof(Key); + auto start_offset = full_key.data() + sec_type_trailing_prefix_size; + const b1::chain_kv::bytes composite_trailing_prim_sec_key {start_offset, start_offset + key_size}; + auto composite_loc = composite_trailing_prim_sec_key.cbegin(); + EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_trailing_prim_sec_key.cend(), sec_key), bad_composite_key_exception, + "DB intrinsic key-value store invariant has changed, extract_key should only fail if the remaining " + "string size is less than: ${s} bytes, which was already verified", ("s", key_size)); + return true; + } + + b1::chain_kv::bytes create_table_key(name scope, name table); + + eosio::session::shared_bytes create_table_key(const eosio::session::shared_bytes& prefix_key); + + bool get_table_key(const b1::chain_kv::bytes& composite_key, name& scope, name& table); + + b1::chain_kv::bytes create_prefix_key(name scope, name table); + + void get_prefix_key(const b1::chain_kv::bytes& composite_key, name& scope, name& table); + + template + rocksdb::Slice prefix_slice(const CharKey& composite_key) { + detail::consistent_keys(); + const auto prefix_size = detail::prefix_size(); + EOS_ASSERT(composite_key.size() >= prefix_size, bad_composite_key_exception, + "Cannot extract a prefix from a key that is not big enough to contain a prefix"); + return rocksdb::Slice(composite_key.data(), prefix_size); + } + + template + rocksdb::Slice prefix_type_slice(const CharKey& composite_key) { + detail::consistent_keys(); + const auto prefix_type_size = detail::prefix_size() + sizeof(key_type); + EOS_ASSERT(composite_key.size() >= prefix_type_size, bad_composite_key_exception, + "Cannot extract a prefix from a key that is not big enough to contain a prefix"); + return rocksdb::Slice(composite_key.data(), prefix_type_size); + } + + template + rocksdb::Slice prefix_primary_to_secondary_slice(const CharKey& composite_key, bool with_primary_key) { + detail::consistent_keys(); + // possibly add the extra_prefix_size (to cover db_type and contract for a full key) + static constexpr auto prefix_size = detail::prefix_size(); + static constexpr auto prefix_type_size = prefix_size + sizeof(key_type); + auto expected_size = prefix_type_size + sizeof(key_type) + (with_primary_key ? sizeof(uint64_t) : 0); + EOS_ASSERT(composite_key.size() >= expected_size, bad_composite_key_exception, + "Cannot extract a primary-to-secondary key prefix from a key that is not big enough to contain a one"); + const char* offset = composite_key.data() + prefix_size; + EOS_ASSERT(key_type{*offset++} == key_type::primary_to_sec, bad_composite_key_exception, + "Cannot extract a primary-to-secondary key prefix from a key that is not of that type"); + const key_type sec_type { *offset }; + EOS_ASSERT(sec_type >= key_type::sec_i64 && sec_type <= key_type::sec_long_double, + bad_composite_key_exception, + "Invariant failure, composite_key is of the correct type, but the format of the key after that is incorrect"); + return rocksdb::Slice(composite_key.data(), expected_size); + } + + key_type extract_key_type(const b1::chain_kv::bytes& composite_key); + + key_type extract_primary_to_sec_key_type(const b1::chain_kv::bytes& composite_key); + + intermittent_decomposed_values get_prefix_thru_key_type(const b1::chain_kv::bytes& composite_key); + + b1::chain_kv::bytes extract_legacy_key(const eosio::session::shared_bytes& complete_key); + + rocksdb::Slice extract_legacy_slice(const eosio::session::shared_bytes& complete_key); + + eosio::session::shared_bytes create_full_key(const b1::chain_kv::bytes& ck, name code); + enum class end_of_prefix { pre_type, at_type, at_prim_to_sec_type, at_prim_to_sec_primary_key }; + eosio::session::shared_bytes create_full_key_prefix(const eosio::session::shared_bytes& ck, end_of_prefix prefix_end); + + struct full_key_data { + char db_type; + std::optional contract; + std::optional legacy_key; + std::optional scope; + std::optional table; + std::optional kt; + rocksdb::Slice type_prefix; + }; + full_key_data parse_full_key(const eosio::session::shared_bytes& full_key); + + template + bool get_primary_key(const CharKey1& full_key, const CharKey2& type_prefix, uint64_t& primary_key) { + detail::consistent_keys(); + // possibly add the extra_prefix_size (to cover db_type and contract for a full key) + const auto type_prefix_size = type_prefix.size(); + const auto prefix_type_size = detail::prefix_size() + sizeof(key_type); + EOS_ASSERT(type_prefix_size == prefix_type_size, bad_composite_key_exception, + "DB intrinsic key-value get_primary_key was passed a type_prefix with key size: ${s1} bytes " + "which is not equal to the expected size: ${s2}", ("s1", type_prefix_size)("s2", prefix_type_size)); + const auto key_size = sizeof(primary_key); + EOS_ASSERT(full_key.size() == type_prefix_size + key_size, bad_composite_key_exception, + "DB intrinsic key-value get_primary_key was passed a full key size: ${s1} bytes that was not exactly " + "${s2} bytes (the size of a primary key) larger than the type_prefix size: ${s3}", + ("s1", full_key.size())("s2", key_size)("s3", type_prefix_size)); + const auto comp = memcmp(type_prefix.data(), full_key.data(), type_prefix_size); + if (comp != 0) { + return false; + } + const key_type actual_kt {*(type_prefix.data() + type_prefix_size - 1)}; + EOS_ASSERT(key_type::primary == actual_kt, bad_composite_key_exception, + "DB intrinsic key-value get_primary_key was passed a key that was the wrong type: ${type}," + " it should have been of type: ${type2}", + ("type", detail::to_string(actual_kt))("type2", detail::to_string(key_type::primary))); + auto start_offset = full_key.data() + type_prefix_size; + const b1::chain_kv::bytes composite_trailing_prim_key {start_offset, start_offset + key_size}; + auto composite_loc = composite_trailing_prim_key.cbegin(); + EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_trailing_prim_key.cend(), primary_key), bad_composite_key_exception, + "DB intrinsic key-value get_primary_key invariant has changed, extract_key should only fail if the " + "remaining string size is less than the sizeof(uint64_t)"); + return true; + } + + eosio::session::shared_bytes create_full_primary_key(name code, name scope, name table, uint64_t primary_key); + eosio::session::shared_bytes create_full_prefix_key(name code, name scope, name table, std::optional kt = std::optional{}); + + template + eosio::session::shared_bytes create_full_secondary_key(name code, name scope, name table, const Key& sec_key, uint64_t primary_key) { + bytes composite_key = create_secondary_key(scope, table, sec_key, primary_key); + return create_full_key(composite_key, code); + } + + template + eosio::session::shared_bytes create_full_prefix_secondary_key(name code, name scope, name table, const Key& sec_key) { + bytes composite_key = create_prefix_secondary_key(scope, table, sec_key); + return create_full_key(composite_key, code); + } + +}}}} // ns eosio::chain::backing_store::db_key_value_format diff --git a/libraries/chain/include/eosio/chain/backing_store/db_key_value_iter_store.hpp b/libraries/chain/include/eosio/chain/backing_store/db_key_value_iter_store.hpp new file mode 100644 index 00000000000..d9f921325ff --- /dev/null +++ b/libraries/chain/include/eosio/chain/backing_store/db_key_value_iter_store.hpp @@ -0,0 +1,160 @@ +#pragma once +#include +#include +#include +#include + +namespace eosio { namespace chain { namespace backing_store { +struct unique_table { + name contract; + name scope; + name table; +}; + +template +struct secondary_key { + int table_ei; + T secondary; + uint64_t primary; + account_name payer; +}; + +inline bool operator<(const unique_table& lhs, const unique_table& rhs) { + return std::tie(lhs.contract, lhs.scope, lhs.table) < std::tie(rhs.contract, rhs.scope, rhs.table); +} + +template +bool operator<(const secondary_key& lhs, const secondary_key& rhs) { + // checking primary second to optimize the search since a given primary key + // cannot have more than one secondary key with the same value (but keeping + // remainder of algorithm for consistency) + return std::tie(lhs.table_ei, lhs.primary, lhs.secondary) < std::tie(rhs.table_ei, rhs.primary, rhs.secondary); + // ignoring payer, it does not contribute to uniqueness +} + +template +class db_key_value_iter_store { + public: + using secondary_obj_type = secondary_key; + using secondary_key_type = SecondaryKey; + + db_key_value_iter_store(){ + _end_iterator_to_table.reserve(8); + _iterator_to_object.reserve(32); + } + + constexpr int invalid_iterator() const { return -1; } + + /// Returns end iterator of the table. + int cache_table( const unique_table& tobj ) { + auto itr = _table_cache.find(tobj); + if( itr != _table_cache.end() ) + return itr->second; + + auto ei = index_to_end_iterator(_end_iterator_to_table.size()); + _end_iterator_to_table.push_back( tobj ); + _table_cache.emplace( tobj, ei ); + return ei; + } + + int get_end_iterator_by_table( const unique_table& tobj )const { + auto itr = _table_cache.find(tobj); + EOS_ASSERT( itr != _table_cache.end(), table_not_in_cache, "an invariant was broken, table should be in cache" ); + return itr->second; + } + + const unique_table* find_table_by_end_iterator( int ei )const { + EOS_ASSERT( ei < invalid_iterator(), invalid_table_iterator, "not an end iterator" ); + auto indx = end_iterator_to_index(ei); + if( indx >= _end_iterator_to_table.size() ) return nullptr; + return &_end_iterator_to_table[indx]; + } + + const unique_table& get_table( const secondary_obj_type& obj )const { + EOS_ASSERT( obj.table_ei < invalid_iterator(), invalid_table_iterator, "not an end iterator" ); + auto indx = end_iterator_to_index(obj.table_ei); + EOS_ASSERT( indx < _end_iterator_to_table.size(), table_not_in_cache, + "an invariant was broken, secondary object references table that is not in cache" ); + return _end_iterator_to_table[indx]; + } + + const secondary_obj_type& get( int iterator ) const { + validate_object_iterator(iterator, "dereference of end iterator"); + // grab a const ref, to ensure that we are returning a reference to the actual object in the vector + const auto& result = _iterator_to_object[iterator]; + EOS_ASSERT( result, table_operation_not_permitted, "dereference of deleted object" ); + return *result; + } + + void remove( int iterator ) { + validate_object_iterator(iterator, "cannot call remove on end iterators"); + + auto optional_obj = _iterator_to_object[iterator]; + if( !optional_obj ) return; + _object_to_iterator.erase( *optional_obj ); + + _iterator_to_object[iterator].reset(); + } + + void swap( int iterator, const secondary_key_type& secondary, account_name payer ) { + validate_object_iterator(iterator, "cannot call swap on end iterators"); + + auto& optional_obj = _iterator_to_object[iterator]; + if( !optional_obj ) return; + // check to see if the object will actually change + if(optional_obj->payer == payer && optional_obj->secondary == secondary) return; + const bool map_key_change = optional_obj->secondary != secondary; + if (map_key_change) { + // edit object and swap out the map entry + _object_to_iterator.erase( *optional_obj ); + optional_obj->secondary = secondary; + } + optional_obj->payer = payer; + + if (map_key_change) { + _object_to_iterator.insert({ *optional_obj, iterator }); + } + } + + int find( const secondary_obj_type& obj ) const { + auto itr = _object_to_iterator.find( obj ); + if( itr != _object_to_iterator.end() ) + return itr->second; + + return invalid_iterator(); + } + + int add( const secondary_obj_type& obj ) { + auto itr = find( obj ); + if( itr != invalid_iterator() ) + return itr; + + EOS_ASSERT( obj.table_ei < invalid_iterator(), invalid_table_iterator, "not an end iterator" ); + const auto indx = end_iterator_to_index(obj.table_ei); + EOS_ASSERT( indx < _end_iterator_to_table.size(), invalid_table_iterator, "an invariant was broken, table should be in cache" ); + _object_to_iterator.insert({ obj, _iterator_to_object.size() }); + _iterator_to_object.emplace_back( obj ); + + return _iterator_to_object.size() - 1; + } + + private: + void validate_object_iterator(int iterator, const char* explanation_for_no_end_iterators) const { + EOS_ASSERT( iterator != invalid_iterator(), invalid_table_iterator, "invalid iterator" ); + EOS_ASSERT( iterator >= 0, table_operation_not_permitted, explanation_for_no_end_iterators ); + EOS_ASSERT( (size_t)iterator < _iterator_to_object.size(), invalid_table_iterator, "iterator out of range" ); + } + + map _table_cache; + vector _end_iterator_to_table; + vector> _iterator_to_object; + map _object_to_iterator; + + /// Precondition: std::numeric_limits::min() < ei < -1 + /// Iterator of -1 is reserved for invalid iterators (i.e. when the appropriate table has not yet been created). + inline size_t end_iterator_to_index( int ei )const { return (-ei - 2); } + /// Precondition: indx < _end_iterator_to_table.size() <= std::numeric_limits::max() + inline int index_to_end_iterator( size_t indx )const { return -(indx + 2); } +}; /// class db_key_value_iter_store + +}}} // namespace eosio::chain::backing_store diff --git a/libraries/chain/include/eosio/chain/backing_store/db_key_value_sec_lookup.hpp b/libraries/chain/include/eosio/chain/backing_store/db_key_value_sec_lookup.hpp new file mode 100644 index 00000000000..a09e8ebf148 --- /dev/null +++ b/libraries/chain/include/eosio/chain/backing_store/db_key_value_sec_lookup.hpp @@ -0,0 +1,639 @@ +#pragma once + +#include +#include + +namespace eosio { namespace chain { namespace backing_store { + + template + struct secondary_helper; + + template<> + struct secondary_helper { + const char* desc() { return "idx64"; } + eosio::session::shared_bytes value(const uint64_t& sec_key, name payer) { + return payer_payload(payer, nullptr, 0).as_payload(); + } + void extract(const eosio::session::shared_bytes& payload, uint64_t& sec_key, name& payer) { + payer_payload pp(payload); + payer = pp.payer; + EOS_ASSERT( pp.value_size == 0, db_rocksdb_invalid_operation_exception, + "Payload should not have anything more than the payer"); + } + constexpr inline uint64_t overhead() { return config::billable_size_v;} + }; + + template<> + struct secondary_helper { + const char* desc() { return "idx128"; } + eosio::session::shared_bytes value(const eosio::chain::uint128_t& sec_key, name payer) { + return payer_payload(payer, nullptr, 0).as_payload(); + } + void extract(const eosio::session::shared_bytes& payload, eosio::chain::uint128_t& sec_key, name& payer) { + payer_payload pp(payload); + payer = pp.payer; + EOS_ASSERT( pp.value_size == 0, db_rocksdb_invalid_operation_exception, + "Payload should not have anything more than the payer"); + } + constexpr inline uint64_t overhead() { return config::billable_size_v;} + }; + + template<> + struct secondary_helper { + const char* desc() { return "idx256"; } + eosio::session::shared_bytes value(const eosio::chain::key256_t& sec_key, name payer) { + return payer_payload(payer, nullptr, 0).as_payload(); + } + void extract(const eosio::session::shared_bytes& payload, eosio::chain::key256_t& sec_key, name& payer) { + payer_payload pp(payload); + payer = pp.payer; + EOS_ASSERT( pp.value_size == 0, db_rocksdb_invalid_operation_exception, + "Payload should not have anything more than the payer"); + } + constexpr inline uint64_t overhead() { return config::billable_size_v;} + }; + + template<> + struct secondary_helper { + const char* desc() { return "idx_double"; } + eosio::session::shared_bytes value(const float64_t& sec_key, name payer) { + constexpr static auto value_size = sizeof(sec_key); + char value_buf[value_size]; + memcpy(value_buf, &sec_key, value_size); + // float64_t stores off the original bit pattern since the key pattern has to lose the differentiation + // between +0.0 and -0.0 + return payer_payload(payer, value_buf, value_size).as_payload(); + } + void extract(const eosio::session::shared_bytes& payload, float64_t& sec_key, name& payer) { + payer_payload pp(payload); + EOS_ASSERT( pp.value_size == sizeof(sec_key), db_rocksdb_invalid_operation_exception, + "Payload does not contain the expected float64_t exact representation"); + payer = pp.payer; + // see note on value(...) method above + memcpy(&sec_key, pp.value, pp.value_size); + } + constexpr inline uint64_t overhead() { return config::billable_size_v;} + }; + + template<> + struct secondary_helper { + const char* desc() { return "idx_long_double"; } + eosio::session::shared_bytes value(const float128_t& sec_key, name payer) { + constexpr static auto value_size = sizeof(sec_key); + // NOTE: this is not written in a way that sorting on value would work, but it is just used for storage + char value_buf[value_size]; + memcpy(value_buf, &sec_key, value_size); + // float128_t stores off the original bit pattern since the key pattern has to lose the differentiation + // between +0.0 and -0.0 + return payer_payload(payer, value_buf, value_size).as_payload(); + } + void extract(const eosio::session::shared_bytes& payload, float128_t& sec_key, name& payer) { + payer_payload pp(payload); + EOS_ASSERT( pp.value_size == sizeof(sec_key), db_rocksdb_invalid_operation_exception, + "Payload does not contain the expected float128_t exact representation"); + payer = pp.payer; + // see note on value(...) method above + memcpy(&sec_key, pp.value, pp.value_size); + } + constexpr inline uint64_t overhead() { return config::billable_size_v;} + }; + + template + class db_key_value_sec_lookup : public db_key_value_any_lookup + { + public: + db_key_value_sec_lookup( db_context& p, session_variant_type& s ) : db_key_value_any_lookup(p, s) {} + + int store( name scope, name table, const account_name& payer, + uint64_t id, const SecondaryKey& secondary ) { + EOS_ASSERT( payer != account_name(), invalid_table_payer, "must specify a valid account to pay for new record" ); + + const sec_pair_bundle secondary_key = get_secondary_slices_in_secondaries(parent.receiver, scope, table, secondary, id); + + add_table_if_needed(secondary_key.full_secondary_key, payer); + + auto old_value = current_session.read(secondary_key.full_secondary_key); + + EOS_ASSERT( !old_value, db_rocksdb_invalid_operation_exception, "db_${d}_store called with pre-existing key", ("d", helper.desc())); + + // identify if this primary key already has a secondary key of this type + auto session_iter = current_session.lower_bound(secondary_key.prefix_primary_to_sec_key); + EOS_ASSERT( !match_prefix(secondary_key.prefix_primary_to_sec_key, session_iter), db_rocksdb_invalid_operation_exception, + "db_${d}_store called, but primary key: ${primary} already has a secondary key of this type", + ("d", helper.desc())("primary", id)); + + set_value(secondary_key.full_secondary_key, helper.value(secondary, payer)); + set_value(secondary_key.full_primary_to_sec_key, useless_value); + + std::string event_id; + if (parent.context.control.get_deep_mind_logger() != nullptr) { + event_id = db_context::table_event(parent.receiver, scope, table, name(id)); + } + + parent.context.update_db_usage( payer, helper.overhead(), backing_store::db_context::secondary_add_trace(parent.context.get_action_id(), std::move(event_id)) ); + + const unique_table t { parent.receiver, scope, table }; + const auto table_ei = iter_store.cache_table(t); + return iter_store.add({ .table_ei = table_ei, .secondary = secondary, .primary = id, .payer = payer }); + } + + void remove( int iterator ) { + const iter_obj& key_store = iter_store.get(iterator); + const unique_table& table = iter_store.get_table(key_store); + EOS_ASSERT( table.contract == parent.receiver, table_access_violation, "db access violation" ); + + const sec_pair_bundle secondary_key = get_secondary_slices_in_secondaries(parent.receiver, table.scope, table.table, key_store.secondary, key_store.primary); + auto old_value = current_session.read(secondary_key.full_secondary_key); + + EOS_ASSERT( old_value, db_rocksdb_invalid_operation_exception, + "invariant failure in db_${d}_remove, iter store found to update but nothing in database", ("d", helper.desc())); + + auto session_iter = current_session.lower_bound(secondary_key.prefix_primary_to_sec_key); + EOS_ASSERT( match_prefix(secondary_key.full_primary_to_sec_key, session_iter), db_rocksdb_invalid_operation_exception, + "db_${d}_remove called, but primary key: ${primary} didn't have a primary to secondary key", + ("d", helper.desc())("primary", key_store.primary)); + + std::string event_id; + if (parent.context.control.get_deep_mind_logger() != nullptr) { + event_id = db_context::table_event(table.contract, table.scope, table.table, name(key_store.primary)); + } + + parent.context.update_db_usage( key_store.payer, -( helper.overhead() ), db_context::secondary_rem_trace(parent.context.get_action_id(), std::move(event_id)) ); + + current_session.erase(secondary_key.full_secondary_key); + current_session.erase(secondary_key.full_primary_to_sec_key); + remove_table_if_empty(secondary_key.full_secondary_key); + iter_store.remove(iterator); + } + + void update( int iterator, account_name payer, const SecondaryKey& secondary ) { + const iter_obj& key_store = iter_store.get(iterator); + const unique_table& table = iter_store.get_table(key_store); + EOS_ASSERT( table.contract == parent.receiver, table_access_violation, "db access violation" ); + + const sec_pair_bundle secondary_key = get_secondary_slices_in_secondaries(parent.receiver, table.scope, table.table, key_store.secondary, key_store.primary); + auto old_value = current_session.read(secondary_key.full_secondary_key); + + secondary_helper helper; + EOS_ASSERT( old_value, db_rocksdb_invalid_operation_exception, + "invariant failure in db_${d}_remove, iter store found to update but nothing in database", ("d", helper.desc())); + + // identify if this primary key already has a secondary key of this type + auto primary_sec_uesless_value = current_session.read(secondary_key.full_primary_to_sec_key); + EOS_ASSERT( primary_sec_uesless_value, db_rocksdb_invalid_operation_exception, + "invariant failure in db_${d}_update, the secondary to primary lookup key was found, but not the " + "primary to secondary lookup key", ("d", helper.desc())); + + if( payer == account_name() ) payer = key_store.payer; + + auto& context = parent.context; + std::string event_id; + if (context.control.get_deep_mind_logger() != nullptr) { + event_id = backing_store::db_context::table_event(table.contract, table.scope, table.table, name(key_store.primary)); + } + + if( key_store.payer != payer ) { + context.update_db_usage( key_store.payer, -helper.overhead(), backing_store::db_context::secondary_update_rem_trace(context.get_action_id(), std::string(event_id)) ); + context.update_db_usage( payer, helper.overhead(), backing_store::db_context::secondary_update_add_trace(context.get_action_id(), std::move(event_id)) ); + } + + // if the secondary value is different, remove the old key and add the new key + if (secondary != key_store.secondary) { + // remove the secondary (to primary) key and the primary to secondary key + current_session.erase(secondary_key.full_secondary_key); + current_session.erase(secondary_key.full_primary_to_sec_key); + + // store the new secondary (to primary) key + const auto new_secondary_keys = db_key_value_format::create_secondary_key_pair(table.scope, table.table, secondary, key_store.primary); + set_value(db_key_value_format::create_full_key(new_secondary_keys.secondary_key, parent.receiver), helper.value(secondary, payer)); + + // store the new primary to secondary key +#warning currently using useless_value to distinguish between no key and no value, eventually will just store empty value + set_value(db_key_value_format::create_full_key(new_secondary_keys.primary_to_secondary_key, parent.receiver), useless_value); + } + else { + // no key values have changed, so can use the old key (and no change to secondary to primary key) + set_value(secondary_key.full_primary_to_sec_key, helper.value(secondary, payer)); + } + iter_store.swap(iterator, secondary, payer); + } + + int find_secondary( name code, name scope, name table, const SecondaryKey& secondary, uint64_t& primary ) { + prefix_bundle secondary_key = get_secondary_slice_in_table(code, scope, table, secondary); + auto session_iter = current_session.lower_bound(secondary_key.full_key); + + // if we don't get any match for this table, then return the invalid iterator + if (!match_prefix(secondary_key.prefix_key, session_iter)) { + // no keys for this entire table + return iter_store.invalid_iterator(); + } + + const unique_table t { code, scope, table }; + const auto table_ei = iter_store.cache_table(t); + // verify if the key that was found contains this secondary key + if (!match_prefix(secondary_key.full_key, session_iter)) { + return table_ei; + } + + const auto full_slice = db_key_value_format::extract_legacy_slice((*session_iter).first); + const auto prefix_secondary_slice = db_key_value_format::extract_legacy_slice(secondary_key.full_key); + const bool valid_key = db_key_value_format::get_trailing_primary_key(full_slice, prefix_secondary_slice, primary); + EOS_ASSERT( valid_key, db_rocksdb_invalid_operation_exception, + "invariant failure in db_${d}_find_secondary, key portions were identical but " + "get_trailing_primary_key indicated that it couldn't extract", ("d", helper.desc())); + + return iter_store.add({ .table_ei = table_ei, .secondary = secondary, .primary = primary, + .payer = payer_payload(*(*session_iter).second).payer }); + } + + int lowerbound_secondary( name code, name scope, name table, SecondaryKey& secondary, uint64_t& primary ) { + return bound_secondary(code, scope, table, bound_type::lower, secondary, primary); + } + + int upperbound_secondary( name code, name scope, name table, SecondaryKey& secondary, uint64_t& primary ) { + return bound_secondary(code, scope, table, bound_type::upper, secondary, primary); + } + + int end_secondary( name code, name scope, name table ) { + return get_end_iter(name{code}, name{scope}, name{table}, iter_store); + } + + int next_secondary( int iterator, uint64_t& primary ) { + if( iterator < iter_store.invalid_iterator() ) return iter_store.invalid_iterator(); // cannot increment past end iterator of index + + const iter_obj& key_store = iter_store.get(iterator); + const unique_table& table = iter_store.get_table(key_store); + + prefix_bundle secondary_key = get_secondary_slice_in_secondaries(table.contract, table.scope, table.table, key_store.secondary, key_store.primary); + auto session_iter = current_session.lower_bound(secondary_key.full_key); + EOS_ASSERT( match(secondary_key.full_key, session_iter), db_rocksdb_invalid_operation_exception, + "invariant failure in db_${d}_next_secondary, found iterator in iter store but didn't find " + "any entry in the database (no secondary keys of this specific type)", ("d", helper.desc())); + ++session_iter; + if (!match_prefix(secondary_key.prefix_key, session_iter)) { + return key_store.table_ei; + } + + SecondaryKey secondary; + const bool valid_key = db_key_value_format::get_trailing_sec_prim_keys((*session_iter).first, secondary_key.prefix_key, secondary, primary); + EOS_ASSERT( valid_key, db_rocksdb_invalid_operation_exception, + "invariant failure in db_${d}_next_secondary, since the type slice matched, the trailing " + "primary key should have been extracted", ("d", helper.desc())); + + return iter_store.add({ .table_ei = key_store.table_ei, .secondary = secondary, .primary = primary, + .payer = payer_payload(*(*session_iter).second).payer }); + } + + int previous_secondary( int iterator, uint64_t& primary ) { + if( iterator < iter_store.invalid_iterator() ) { + const backing_store::unique_table* table = iter_store.find_table_by_end_iterator(iterator); + EOS_ASSERT( table, invalid_table_iterator, "not a valid end iterator" ); + constexpr static auto kt = db_key_value_format::derive_secondary_key_type(); + const bytes legacy_type_key = db_key_value_format::create_prefix_type_key(table->scope, table->table, kt); + const shared_bytes type_prefix = db_key_value_format::create_full_key(legacy_type_key, table->contract); + const shared_bytes next_type_prefix = type_prefix.next(); + auto session_iter = current_session.lower_bound(next_type_prefix); + --session_iter; // move back to the last secondary key of this type (if there is one) + // check if there is any secondary key of this type + if (!match_prefix(type_prefix, session_iter)) { + return iter_store.invalid_iterator(); + } + + auto found_keys = find_matching_sec_prim(*session_iter, type_prefix); + if (!found_keys) { + return iter_store.invalid_iterator(); + } + primary = std::get(*found_keys); + return iter_store.add({ .table_ei = iterator, .secondary = std::move(std::get(*found_keys)), + .primary = std::move(std::get(*found_keys)), + .payer = std::move(std::get(*found_keys))}); + } + + const iter_obj& key_store = iter_store.get(iterator); + const unique_table& table = iter_store.get_table(key_store); + + prefix_bundle secondary_key = get_secondary_slice_in_secondaries(table.contract, table.scope, table.table, key_store.secondary, key_store.primary); + auto session_iter = current_session.lower_bound(secondary_key.full_key); + EOS_ASSERT( match(secondary_key.full_key, session_iter), db_rocksdb_invalid_operation_exception, + "invariant failure in db_${d}_next_secondary, found iterator in iter store but didn't find its " + "matching entry in the database", ("d", helper.desc())); + --session_iter; + if (!match_prefix(secondary_key.prefix_key, session_iter)) { + return key_store.table_ei; + } + + auto found_keys = find_matching_sec_prim(*session_iter, secondary_key.prefix_key); + EOS_ASSERT( found_keys, db_rocksdb_invalid_operation_exception, + "invariant failure in db_${d}_previous_secondary, since the type slice matched, the trailing " + "primary key should have been extracted", ("d", helper.desc())); + primary = std::get(*found_keys); + + return iter_store.add({ .table_ei = key_store.table_ei, .secondary = std::move(std::get(*found_keys)), + .primary = primary, .payer = std::move(std::get(*found_keys)) }); + } + + int find_primary( name code, name scope, name table, SecondaryKey& secondary, uint64_t primary ) { + const bytes legacy_prim_to_sec_key = db_key_value_format::create_prefix_primary_to_secondary_key(scope, table, primary); + const shared_bytes key = db_key_value_format::create_full_key(legacy_prim_to_sec_key, code); + const shared_bytes prefix = db_key_value_format::create_full_key_prefix(key, end_of_prefix::pre_type); + auto session_iter = current_session.lower_bound(key); + // check if nothing remains in the primary table space + if (!match_prefix(prefix, session_iter)) { + return iter_store.invalid_iterator(); + } + + const unique_table t { code, scope, table }; + const auto table_ei = iter_store.cache_table(t); + + // if this is not of the same type and primary key + if (!backing_store::db_key_value_format::get_trailing_secondary_key((*session_iter).first, key, secondary)) { + // since this is not the desired secondary key entry, reject it + return table_ei; + } + + const bytes legacy_secondary_key = db_key_value_format::create_secondary_key(scope, table, secondary, primary); + const shared_bytes secondary_key = db_key_value_format::create_full_key(legacy_secondary_key, code); + const auto old_value = current_session.read(secondary_key); + EOS_ASSERT( old_value, db_rocksdb_invalid_operation_exception, + "invariant failure in db_${d}_previous_secondary, found primary to secondary key in database but " + "not the secondary to primary key", ("d", helper.desc())); + EOS_ASSERT( old_value->data() != nullptr, db_rocksdb_invalid_operation_exception, + "invariant failure in db_${d}_previous_secondary, empty value", ("d", helper.desc())); + EOS_ASSERT( old_value->size(), db_rocksdb_invalid_operation_exception, + "invariant failure in db_${d}_previous_secondary, value has zero size", ("d", helper.desc())); + payer_payload pp{*old_value}; + + return iter_store.add({ .table_ei = table_ei, .secondary = secondary, .primary = primary, + .payer = pp.payer}); + } + + int lowerbound_primary( name code, name scope, name table, uint64_t primary ) { + return bound_primary(code, scope, table, primary, bound_type::lower); + } + + int upperbound_primary( name code, name scope, name table, uint64_t primary ) { + return bound_primary(code, scope, table, primary, bound_type::upper); + } + + int next_primary( int iterator, uint64_t& primary ) { + if( iterator < iter_store.invalid_iterator() ) return iter_store.invalid_iterator(); // cannot increment past end iterator of index + + const iter_obj& key_store = iter_store.get(iterator); + const unique_table& table = iter_store.get_table(key_store); + + const bytes prim_to_sec_key = + db_key_value_format::create_primary_to_secondary_key(table.scope, table.table, key_store.primary, key_store.secondary); + const shared_bytes key = db_key_value_format::create_full_key(prim_to_sec_key, table.contract); + const shared_bytes sec_type_prefix = db_key_value_format::create_full_key_prefix(key, end_of_prefix::at_prim_to_sec_type); + auto session_iter = current_session.lower_bound(key); + // This key has to exist, since it is cached in our iterator store + EOS_ASSERT( match(key, session_iter), db_rocksdb_invalid_operation_exception, + "invariant failure in db_${d}_next_primary, found iterator in iter store but didn't find its " + "matching entry in the database", ("d", helper.desc())); + ++session_iter; + if (!match_prefix(sec_type_prefix, session_iter)) { + return key_store.table_ei; + } + + auto found_keys = find_matching_prim_sec(*session_iter, sec_type_prefix); + EOS_ASSERT( found_keys, db_rocksdb_invalid_operation_exception, + "invariant failure in db_${d}_next_primary, since the type slice matched, the trailing " + "keys should have been extracted", ("d", helper.desc())); + primary = std::get(*found_keys); + return iter_store.add({ .table_ei = key_store.table_ei, .secondary = std::move(std::get(*found_keys)), + .primary = std::move(std::get(*found_keys)), + .payer = std::move(std::get(*found_keys))}); + } + + int previous_primary( int iterator, uint64_t& primary ) { + if( iterator < iter_store.invalid_iterator() ) { + const backing_store::unique_table* table = iter_store.find_table_by_end_iterator(iterator); + EOS_ASSERT( table, invalid_table_iterator, "not a valid end iterator" ); + const bytes types_key = db_key_value_format::create_prefix_primary_to_secondary_key(table->scope, table->table); + + // create the key pointing to the start of the primary to secondary keys of this type, and then increment it to be past the end + const shared_bytes type_prefix = db_key_value_format::create_full_key(types_key, table->contract); + const shared_bytes next_type_prefix = type_prefix.next(); + auto session_iter = current_session.lower_bound(next_type_prefix); //upperbound of this primary_to_sec key type + --session_iter; // move back to the last primary key of this type (if there is one) + // check if no keys exist for this type + if (!match_prefix(type_prefix, session_iter)) { + return iter_store.invalid_iterator(); + } + auto found_keys = find_matching_prim_sec(*session_iter, type_prefix); + if (!found_keys) { + return iter_store.invalid_iterator(); + } + primary = std::move(std::get(*found_keys)); + return iter_store.add({ .table_ei = iterator, .secondary = std::move(std::get(*found_keys)), + .primary = std::move(std::get(*found_keys)), + .payer = std::move(std::get(*found_keys))}); + } + + const iter_obj& key_store = iter_store.get(iterator); + const unique_table& table = iter_store.get_table(key_store); + + const bytes prim_to_sec_key = + db_key_value_format::create_primary_to_secondary_key(table.scope, table.table, key_store.primary, key_store.secondary); + const shared_bytes key = db_key_value_format::create_full_key(prim_to_sec_key, table.contract); + const shared_bytes sec_type_prefix = db_key_value_format::create_full_key_prefix(key, end_of_prefix::at_prim_to_sec_type); + auto session_iter = current_session.lower_bound(key); + EOS_ASSERT( match(key, session_iter), db_rocksdb_invalid_operation_exception, + "invariant failure in db_${d}_previous_primary, found iterator in iter store but didn't find its " + "matching entry in the database", ("d", helper.desc())); + --session_iter; + // determine if we are still in the range of this primary_to_sec key type + if (!match_prefix(sec_type_prefix, session_iter)) { + return iter_store.invalid_iterator(); + } + + auto found_keys = find_matching_prim_sec(*session_iter, sec_type_prefix); + if (!found_keys) { + return iter_store.invalid_iterator(); + } + primary = std::get(*found_keys); + return iter_store.add({ .table_ei = key_store.table_ei, .secondary = std::move(std::get(*found_keys)), + .primary = std::move(std::get(*found_keys)), + .payer = std::move(std::get(*found_keys))}); + } + + void get( int iterator, uint64_t& primary, SecondaryKey& secondary ) { + const iter_obj& key_store = iter_store.get(iterator); + + primary = key_store.primary; + secondary = key_store.secondary; + } + + struct sec_pair_bundle { + sec_pair_bundle(const b1::chain_kv::bytes& composite_sec_key, const b1::chain_kv::bytes& composite_prim_to_sec_key, name code) + : full_secondary_key(db_key_value_format::create_full_key(composite_sec_key, code)), + full_primary_to_sec_key(db_key_value_format::create_full_key(composite_prim_to_sec_key, code)), + prefix_secondary_key(db_key_value_format::create_full_key_prefix(full_secondary_key, end_of_prefix::at_type)), + prefix_primary_to_sec_key(db_key_value_format::create_full_key_prefix(full_primary_to_sec_key, end_of_prefix::at_prim_to_sec_primary_key)) { + + } + + shared_bytes full_secondary_key; + shared_bytes full_primary_to_sec_key; + shared_bytes prefix_secondary_key; // points to the type prefix for composite_key + shared_bytes prefix_primary_to_sec_key; // points to the primary key in full_primary_to_sec_key (all except the secondary key) + }; + + // gets a prefix that allows for only this secondary key iterators, and the primary to secondary key, with a prefix key to the secondary key type + static sec_pair_bundle get_secondary_slices_in_secondaries(name code, name scope, name table, const SecondaryKey& secondary, uint64_t primary) { + auto secondary_keys = db_key_value_format::create_secondary_key_pair(scope, table, secondary, primary); + return { secondary_keys.secondary_key, secondary_keys.primary_to_secondary_key, code }; + } + + // gets a specific secondary key without the trailing primary key, and will allow for only secondaries of this type (prefix) + static prefix_bundle get_secondary_slice_in_secondaries(name code, name scope, name table, const SecondaryKey& secondary) { + bytes secondary_key = db_key_value_format::create_prefix_secondary_key(scope, table, secondary); + return { secondary_key, end_of_prefix::at_type, code }; + } + + // gets a specific secondary and primary keys, and will allow for only secondaries of this type (prefix) + static prefix_bundle get_secondary_slice_in_secondaries(name code, name scope, name table, const SecondaryKey& secondary, uint64_t primary) { + bytes secondary_key = db_key_value_format::create_secondary_key(scope, table, secondary, primary); + return { secondary_key, end_of_prefix::at_type, code }; + } + + // gets a prefix that allows for a specific secondary key without the trailing primary key, but will allow for all iterators in the table + static prefix_bundle get_secondary_slice_in_table(name code, name scope, name table, const SecondaryKey& secondary) { + bytes secondary_key = db_key_value_format::create_prefix_secondary_key(scope, table, secondary); + return { secondary_key, end_of_prefix::pre_type, code }; + } + + void set_value(const shared_bytes& full_key, const shared_bytes& payload) { + current_session.write(full_key, payload); + } + + private: + using found_keys = std::tuple; + struct fk_index { + static constexpr std::size_t secondary = 0; + static constexpr std::size_t primary = 1; + static constexpr std::size_t payer = 2; + }; + + template + std::optional find_matching_sec_prim(const std::pair >& found_kv, + const CharCont& prefix) { + SecondaryKey secondary_key; + uint64_t primary_key; + if (!backing_store::db_key_value_format::get_trailing_sec_prim_keys(found_kv.first, prefix, secondary_key, primary_key)) { + // since this is not the desired secondary key entry, reject it + return {}; + } + + EOS_ASSERT( found_kv.second, db_rocksdb_invalid_operation_exception, + "invariant failure in db_${d}_previous_secondary, the secondary key was found but it had no value " + "(the payer) stored for it", ("d", helper.desc())); + account_name payer = payer_payload(*found_kv.second).payer; + return found_keys{ std::move(secondary_key), std::move(primary_key), std::move(payer) }; + } + + template + std::optional find_matching_prim_sec(const std::pair>& found_kv, + const shared_bytes& prefix) { + SecondaryKey secondary_key; + uint64_t primary_key; + if (!backing_store::db_key_value_format::get_trailing_prim_sec_keys(found_kv.first, prefix, primary_key, secondary_key)) { + // since this is not the desired secondary key entry, reject it + return {}; + } + + auto full_secondary_key = backing_store::db_key_value_format::create_secondary_key_from_primary_to_sec_key(found_kv.first); + auto session_iter = current_session.read(full_secondary_key); + EOS_ASSERT( match(found_kv.first, session_iter), db_rocksdb_invalid_operation_exception, + "invariant failure in find_matching_prim_sec, the primary to sec key was found in the database, " + "but no secondary to primary key"); + EOS_ASSERT( (*session_iter)->second, db_rocksdb_invalid_operation_exception, + "invariant failure in find_matching_prim_sec, the secondary key was found but it had no value " + "(the payer) stored for it"); + account_name payer = payer_payload(*(*session_iter)->second).payer; + return found_keys{ std::move(secondary_key), std::move(primary_key), std::move(payer) }; + } + + enum class bound_type { lower, upper }; + const char* as_string(bound_type bt) { return (bt == bound_type::upper) ? "upper" : "lower"; } + int bound_secondary( name code, name scope, name table, bound_type bt, SecondaryKey& secondary, uint64_t& primary ) { + prefix_bundle secondary_key = get_secondary_slice_in_table(code, scope, table, secondary); + auto session_iter = current_session.lower_bound(secondary_key.full_key); + // setting the "key space" to be the whole table, so that we either get a match or another key for this table + if (!match_prefix(secondary_key.prefix_key, session_iter)) { + // no keys for this entire table + return iter_store.invalid_iterator(); + } + + const unique_table t { code, scope, table }; + const auto table_ei = iter_store.cache_table(t); + const auto type_prefix = db_key_value_format::create_full_key_prefix(secondary_key.full_key, end_of_prefix::at_type); + // verify if the key that was found contains a secondary key of this type + if (!match_prefix(type_prefix, session_iter)) { + return table_ei; + } + else if (bt == bound_type::upper) { + // need to advance till we get to a different secondary key + while (match_prefix(secondary_key.full_key, session_iter)) { + ++session_iter; + if (!match_prefix(type_prefix, session_iter)) { + // no more secondary keys of this type + return table_ei; + } + } + } + + SecondaryKey secondary_ret {}; + uint64_t primary_ret = 0; + const bool valid_key = db_key_value_format::get_trailing_sec_prim_keys( + (*session_iter).first, type_prefix, secondary_ret, primary_ret); + EOS_ASSERT( valid_key, db_rocksdb_invalid_operation_exception, + "invariant failure in db_${d}_${bt_str}bound_secondary, since the type slice matched, the trailing " + "keys should have been extracted", ("d", helper.desc())("bt_str",as_string(bt))); + + // since this method cannot control that the input for primary and secondary having overlapping memory + // setting primary 1st and secondary 2nd to maintain the same results as chainbase + primary = primary_ret; + secondary = secondary_ret; + + return iter_store.add({ .table_ei = table_ei, .secondary = secondary_ret, .primary = primary_ret, + .payer = payer_payload(*(*session_iter).second).payer }); + } + + int bound_primary( name code, name scope, name table, uint64_t primary, bound_type bt ){ + const bytes prim_to_sec_key = db_key_value_format::create_prefix_primary_to_secondary_key(scope, table, primary); + const shared_bytes key = db_key_value_format::create_full_key(prim_to_sec_key, code); + // use the primary to secondary key type to only retrieve secondary keys of SecondaryKey type + const shared_bytes sec_type_prefix = db_key_value_format::create_full_key_prefix(key, end_of_prefix::at_prim_to_sec_type); + auto session_iter = current_session.lower_bound(key); + // check if nothing remains in the table database + if (!match_prefix(sec_type_prefix, session_iter)) { + return iter_store.invalid_iterator(); + } + + const unique_table t { code, scope, table }; + const auto table_ei = iter_store.cache_table(t); + + if (bt == bound_type::upper && match_prefix(key, session_iter)) { + ++session_iter; + if (!match_prefix(sec_type_prefix, session_iter)) { + return table_ei; + } + } + + auto found_keys = find_matching_prim_sec(*session_iter, sec_type_prefix); + EOS_ASSERT( found_keys, db_rocksdb_invalid_operation_exception, + "invariant failure in db_${d}_${bt_str}bound_secondary, since the type slice matched, the trailing " + "keys should have been extracted", ("d", helper.desc())("bt_str",as_string(bt))); + + return iter_store.add({ .table_ei = table_ei, .secondary = std::move(std::get(*found_keys)), + .primary = std::move(std::get(*found_keys)), + .payer = std::move(std::get(*found_keys))}); + } + + db_key_value_iter_store iter_store; + using iter_obj = typename db_key_value_iter_store::secondary_obj_type; + secondary_helper helper; + }; + +}}} // ns eosio::chain::backing_store diff --git a/libraries/chain/include/eosio/chain/backing_store/db_secondary_key_helper.hpp b/libraries/chain/include/eosio/chain/backing_store/db_secondary_key_helper.hpp new file mode 100644 index 00000000000..8e6aab221d3 --- /dev/null +++ b/libraries/chain/include/eosio/chain/backing_store/db_secondary_key_helper.hpp @@ -0,0 +1,67 @@ +#pragma once +#include +#include +#include +#include + +namespace eosio { namespace chain { namespace backing_store { + +template +struct array_size; + +template +struct array_size< std::array > { + static constexpr size_t size = N; +}; + +template +class db_secondary_key_helper; + +template +class db_secondary_key_helper::type>::value>::type > +{ + public: + typedef SecondaryKey secondary_key_type; + + static void set(secondary_key_type& sk_in_table, const secondary_key_type& sk_from_wasm) { + sk_in_table = sk_from_wasm; + } + + static void get(secondary_key_type& sk_from_wasm, const secondary_key_type& sk_in_table ) { + sk_from_wasm = sk_in_table; + } + + static auto create_tuple(const table_id_object& tab, const secondary_key_type& secondary) { + return boost::make_tuple( tab.id, secondary ); + } +}; + +template +class db_secondary_key_helper::type>::value && + std::is_pointer::type>::value>::type > +{ + public: + typedef SecondaryKey secondary_key_type; + typedef SecondaryKeyProxy secondary_key_proxy_type; + typedef SecondaryKeyProxyConst secondary_key_proxy_const_type; + + static constexpr size_t N = array_size::size; + + static void set(secondary_key_type& sk_in_table, secondary_key_proxy_const_type sk_from_wasm) { + std::copy(sk_from_wasm, sk_from_wasm + N, sk_in_table.begin()); + } + + static void get(secondary_key_proxy_type sk_from_wasm, const secondary_key_type& sk_in_table) { + std::copy(sk_in_table.begin(), sk_in_table.end(), sk_from_wasm); + } + + static auto create_tuple(const table_id_object& tab, secondary_key_proxy_const_type sk_from_wasm) { + secondary_key_type secondary; + std::copy(sk_from_wasm, sk_from_wasm + N, secondary.begin()); + return boost::make_tuple( tab.id, secondary ); + } +}; + +}}} // namespace eosio::chain::backing_store diff --git a/libraries/chain/include/eosio/chain/backing_store/kv_context.hpp b/libraries/chain/include/eosio/chain/backing_store/kv_context.hpp new file mode 100644 index 00000000000..65d758a4475 --- /dev/null +++ b/libraries/chain/include/eosio/chain/backing_store/kv_context.hpp @@ -0,0 +1,129 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace chainbase { +class database; +} + +namespace b1::chain_kv { + struct database; + class undo_stack; +} + +namespace eosio { namespace chain { + + class apply_context; + + inline constexpr name kvram_id = "eosio.kvram"_n; + + enum class kv_it_stat { + iterator_ok = 0, // Iterator is positioned at a key-value pair + iterator_erased = -1, // The key-value pair that the iterator used to be positioned at was erased + iterator_end = -2, // Iterator is out-of-bounds + }; + + struct kv_iterator { + virtual ~kv_iterator() {} + + virtual bool is_kv_chainbase_context_iterator() const = 0; + virtual bool is_kv_rocksdb_context_iterator() const = 0; + virtual kv_it_stat kv_it_status() = 0; + virtual int32_t kv_it_compare(const kv_iterator& rhs) = 0; + virtual int32_t kv_it_key_compare(const char* key, uint32_t size) = 0; + virtual kv_it_stat kv_it_move_to_end() = 0; + virtual kv_it_stat kv_it_next(uint32_t* found_key_size, uint32_t* found_value_size) = 0; + virtual kv_it_stat kv_it_prev(uint32_t* found_key_size, uint32_t* found_value_size) = 0; + virtual kv_it_stat kv_it_lower_bound(const char* key, uint32_t size, + uint32_t* found_key_size, uint32_t* found_value_size) = 0; + virtual kv_it_stat kv_it_key(uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) = 0; + virtual kv_it_stat kv_it_value(uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) = 0; + virtual std::optional kv_it_payer() = 0; + }; + + struct kv_resource_trace { + enum class operation { + create = 0, // The operation represents the creation of a key-value pair + update = 1, // The operation represents the update of a key-value pair + erase = 2, // The operation represents the deletion of a key-value pair + }; + + kv_resource_trace(const char* key, uint32_t key_size, operation op): key(key, key_size), op(op) {} + + std::string_view key; + operation op; + + const char* op_to_string() const { + if (op == operation::create) return "create"; + if (op == operation::update) return "update"; + if (op == operation::erase) return "erase"; + + return "unknown"; + } + }; + + struct kv_resource_manager { + void update_table_usage(account_name payer, int64_t delta, const kv_resource_trace& trace) { return _update_table_usage(*_context, delta, trace, payer); } + apply_context* _context; + uint64_t billable_size; + void (*_update_table_usage)(apply_context&, int64_t delta, const kv_resource_trace& trace, account_name payer); + }; + + kv_resource_manager create_kv_resource_manager(apply_context& context); + + struct kv_context { + virtual ~kv_context() {} + + virtual int64_t kv_erase(uint64_t contract, const char* key, uint32_t key_size) = 0; + virtual int64_t kv_set(uint64_t contract, const char* key, uint32_t key_size, const char* value, + uint32_t value_size, account_name payer) = 0; + virtual bool kv_get(uint64_t contract, const char* key, uint32_t key_size, uint32_t& value_size) = 0; + virtual uint32_t kv_get_data(uint32_t offset, char* data, uint32_t data_size) = 0; + + virtual std::unique_ptr kv_it_create(uint64_t contract, const char* prefix, uint32_t size) = 0; + + protected: + // Updates resource usage for payer and returns resource delta + template + int64_t create_table_usage(Resource_manager& resource_manager, const account_name& payer, const char* key, const uint32_t key_size, const uint32_t value_size) { + const int64_t resource_delta = (static_cast(resource_manager.billable_size) + key_size + value_size); + resource_manager.update_table_usage(payer, resource_delta, kv_resource_trace(key, key_size, kv_resource_trace::operation::create)); + return resource_delta; + } + + template + int64_t erase_table_usage(Resource_manager& resource_manager, const account_name& payer, const char* key, const uint32_t key_size, const uint32_t value_size) { + const int64_t resource_delta = -(static_cast(resource_manager.billable_size) + key_size + value_size); + resource_manager.update_table_usage(payer, resource_delta, kv_resource_trace(key, key_size, kv_resource_trace::operation::erase)); + return resource_delta; + } + + template + int64_t update_table_usage(Resource_manager& resource_manager, const account_name& old_payer, const account_name& new_payer, const char* key, const uint32_t key_size, const uint32_t old_value_size, const uint32_t new_value_size) { + // 64-bit arithmetic cannot overflow, because both the key and value are limited to 32-bits + const int64_t old_size = key_size + old_value_size; + const int64_t new_size = key_size + new_value_size; + const int64_t resource_delta = new_size - old_size; + + if (old_payer != new_payer) { + // refund the existing payer + resource_manager.update_table_usage(old_payer, -(old_size + resource_manager.billable_size), kv_resource_trace(key, key_size, kv_resource_trace::operation::update)); + // charge the new payer for full amount + resource_manager.update_table_usage(new_payer, new_size + resource_manager.billable_size, kv_resource_trace(key, key_size, kv_resource_trace::operation::update)); + } else if (old_size != new_size) { + // adjust delta for the existing payer + resource_manager.update_table_usage(new_payer, resource_delta, kv_resource_trace(key, key_size, kv_resource_trace::operation::update)); + } // No need for a final "else" as usage does not change + + return resource_delta; + } + }; + + std::unique_ptr create_kv_chainbase_context(chainbase::database& db, name receiver, + kv_resource_manager resource_manager, const kv_database_config& limits); + +}} // namespace eosio::chain diff --git a/libraries/chain/include/eosio/chain/backing_store/kv_context_chainbase.hpp b/libraries/chain/include/eosio/chain/backing_store/kv_context_chainbase.hpp new file mode 100644 index 00000000000..8397caa1052 --- /dev/null +++ b/libraries/chain/include/eosio/chain/backing_store/kv_context_chainbase.hpp @@ -0,0 +1,277 @@ +#include +#include +#include +#include + +namespace eosio { namespace chain { + + struct kv_iterator_chainbase : kv_iterator { + using index_type = std::decay_t().get_index())>; + using tracker_type = std::decay_t().get_mutable_index().track_removed())>; + + chainbase::database& db; + const index_type& idx = db.get_index(); + tracker_type& tracker; + uint32_t& itr_count; + name contract; + std::vector prefix; + const kv_object* current = nullptr; + + kv_iterator_chainbase(chainbase::database& db, tracker_type& tracker, uint32_t& itr_count, name contract, std::vector prefix) + : db{ db }, tracker{ tracker }, itr_count(itr_count), contract{ contract }, prefix{ std::move(prefix) } { + + ++itr_count; + } + + ~kv_iterator_chainbase() override { --itr_count; } + + bool is_kv_chainbase_context_iterator() const override { return true; } + bool is_kv_rocksdb_context_iterator() const override { return false; } + + template + kv_it_stat move_to(const It& it, uint32_t* found_key_size, uint32_t* found_value_size) { + if (it != idx.end() && it->contract == contract && + it->kv_key.size() >= prefix.size() && !memcmp(it->kv_key.data(), prefix.data(), prefix.size())) { + current = &*it; + *found_key_size = current->kv_key.size(); + *found_value_size = current->kv_value.size(); + return kv_it_stat::iterator_ok; + } else { + current = nullptr; + *found_key_size = 0; + *found_value_size = 0; + return kv_it_stat::iterator_end; + } + } + + kv_it_stat kv_it_status() override { + if (!current) + return kv_it_stat::iterator_end; + else if (!tracker.is_removed(*current)) + return kv_it_stat::iterator_ok; + else + return kv_it_stat::iterator_erased; + } + + int32_t kv_it_compare(const kv_iterator& rhs) override { + EOS_ASSERT(rhs.is_kv_chainbase_context_iterator(), kv_bad_iter, "Incompatible key-value iterators"); + auto& r = static_cast(rhs); + + EOS_ASSERT(contract == r.contract, kv_bad_iter, "Incompatible key-value iterators"); + EOS_ASSERT(!current || !tracker.is_removed(*current), kv_bad_iter, "Iterator to erased element"); + EOS_ASSERT(!r.current || !tracker.is_removed(*r.current), kv_bad_iter, "Iterator to erased element"); + if (!r.current) { + if (!current) + return 0; + else { + return -1; + } + } + if (!current) { + return 1; + } + + return compare_blob(current->kv_key, r.current->kv_key); + } + + int32_t kv_it_key_compare(const char* key, uint32_t size) override { + if (!current) + return 1; + EOS_ASSERT(!tracker.is_removed(*current), kv_bad_iter, "Iterator to erased element"); + return compare_blob(current->kv_key, std::string_view{ key, size }); + } + + kv_it_stat kv_it_move_to_end() override { + current = nullptr; + return kv_it_stat::iterator_end; + } + + kv_it_stat kv_it_next(uint32_t* found_key_size, uint32_t* found_value_size) override { + if (current) { + EOS_ASSERT(!tracker.is_removed(*current), kv_bad_iter, "Iterator to erased element"); + auto it = idx.iterator_to(*current); + ++it; + return move_to(it, found_key_size, found_value_size); + } + return move_to(idx.lower_bound(boost::make_tuple(contract, prefix)), found_key_size, found_value_size); + } + + kv_it_stat kv_it_prev(uint32_t* found_key_size, uint32_t* found_value_size) override { + std::decay_t it; + if (current) { + EOS_ASSERT(!tracker.is_removed(*current), kv_bad_iter, "Iterator to erased element"); + it = idx.iterator_to(*current); + } else + it = idx.upper_bound(boost::make_tuple(contract, blob_prefix{{prefix.data(), prefix.size()}})); + if (it != idx.begin()) + return move_to(--it, found_key_size, found_value_size); + current = nullptr; + *found_key_size = 0; + *found_value_size = 0; + return kv_it_stat::iterator_end; + } + + kv_it_stat kv_it_lower_bound(const char* key, uint32_t size, uint32_t* found_key_size, uint32_t* found_value_size) override { + auto clamped_key = std::max(std::string_view{ key, size }, std::string_view{ prefix.data(), prefix.size() }, unsigned_blob_less{}); + return move_to(idx.lower_bound(boost::make_tuple(contract, clamped_key)), found_key_size, found_value_size); + } + + kv_it_stat kv_it_key(uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) override { + if (!current) { + actual_size = 0; + return kv_it_stat::iterator_end; + } + EOS_ASSERT(!tracker.is_removed(*current), kv_bad_iter, "Iterator to erased element"); + if (offset < current->kv_key.size()) + memcpy(dest, current->kv_key.data() + offset, std::min((size_t)size, current->kv_key.size() - offset)); + actual_size = current->kv_key.size(); + return kv_it_stat::iterator_ok; + } + + kv_it_stat kv_it_value(uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) override { + if (!current) { + actual_size = 0; + return kv_it_stat::iterator_end; + } + EOS_ASSERT(!tracker.is_removed(*current), kv_bad_iter, "Iterator to erased element"); + if (offset < current->kv_value.size()) + memcpy(dest, current->kv_value.data() + offset, std::min((size_t)size, current->kv_value.size() - offset)); + actual_size = current->kv_value.size(); + return kv_it_stat::iterator_ok; + } + + std::optional kv_it_payer() override { + if (!current) return {}; + return current->payer; + } + }; // kv_iterator_chainbase + + template + struct kv_context_chainbase : kv_context { + using tracker_type = std::decay_t().get_mutable_index().track_removed())>; + chainbase::database& db; + tracker_type tracker = db.get_mutable_index().track_removed(); + name receiver; + Resource_manager resource_manager; + const kv_database_config& limits; + uint32_t num_iterators = 0; + std::optional temp_data_buffer; + + kv_context_chainbase(chainbase::database& db, name receiver, + Resource_manager resource_manager, const kv_database_config& limits) + : db{ db }, receiver{ receiver }, resource_manager{ resource_manager }, limits{limits} {} + + int64_t kv_erase(uint64_t contract, const char* key, uint32_t key_size) override { + EOS_ASSERT(name{ contract } == receiver, table_operation_not_permitted, "Can not write to this key"); + temp_data_buffer.reset(); + auto* kv = db.find(boost::make_tuple(name{ contract }, std::string_view{ key, key_size })); + if (!kv) + return 0; + const int64_t resource_delta = erase_table_usage(resource_manager, kv->payer, key, kv->kv_key.size(), kv->kv_value.size()); + + if (auto dm_logger = resource_manager._context->control.get_deep_mind_logger()) { + fc_dlog(*dm_logger, "KV_OP REM ${action_id} ${db} ${payer} ${key} ${odata}", + ("action_id", resource_manager._context->get_action_id()) + ("contract", name{ contract }) + ("payer", kv->payer) + ("key", fc::to_hex(kv->kv_key.data(), kv->kv_key.size())) + ("odata", fc::to_hex(kv->kv_value.data(), kv->kv_value.size())) + ); + } + + tracker.remove(*kv); + return resource_delta; + } + + int64_t kv_set(uint64_t contract, const char* key, uint32_t key_size, const char* value, + uint32_t value_size, account_name payer) override { + EOS_ASSERT(name{ contract } == receiver, table_operation_not_permitted, "Can not write to this key"); + EOS_ASSERT(key_size <= limits.max_key_size, kv_limit_exceeded, "Key too large"); + EOS_ASSERT(value_size <= limits.max_value_size, kv_limit_exceeded, "Value too large"); + + temp_data_buffer.reset(); + auto* kv = db.find( + boost::make_tuple(name{ contract }, std::string_view{ key, key_size })); + if (kv) { + const auto resource_delta = update_table_usage(resource_manager, kv->payer, payer, key, key_size, kv->kv_value.size(), value_size); + + if (auto dm_logger = resource_manager._context->control.get_deep_mind_logger()) { + fc_dlog(*dm_logger, "KV_OP UPD ${action_id} ${db} ${payer} ${key} ${odata}:${ndata}", + ("action_id", resource_manager._context->get_action_id()) + ("contract", name{ contract }) + ("payer", payer) + ("key", fc::to_hex(kv->kv_key.data(), kv->kv_key.size())) + ("odata", fc::to_hex(kv->kv_value.data(), kv->kv_value.size())) + ("ndata", fc::to_hex(value, value_size)) + ); + } + + db.modify(*kv, [&](auto& obj) { + obj.kv_value.assign(value, value_size); + obj.payer = payer; + }); + return resource_delta; + } else { + const int64_t resource_delta = create_table_usage(resource_manager, payer, key, key_size, value_size); + db.create([&](auto& obj) { + obj.contract = name{ contract }; + obj.kv_key.assign(key, key_size); + obj.kv_value.assign(value, value_size); + obj.payer = payer; + }); + + if (auto dm_logger = resource_manager._context->control.get_deep_mind_logger()) { + fc_dlog(*dm_logger, "KV_OP INS ${action_id} ${db} ${payer} ${key} ${ndata}", + ("action_id", resource_manager._context->get_action_id()) + ("contract", name{ contract }) + ("payer", payer) + ("key", fc::to_hex(key, key_size)) + ("ndata", fc::to_hex(value, value_size)) + ); + } + + return resource_delta; + } + } + + bool kv_get(uint64_t contract, const char* key, uint32_t key_size, uint32_t& value_size) override { + auto* kv = db.find(boost::make_tuple(name{ contract }, std::string_view{ key, key_size })); + if (kv) { + temp_data_buffer.emplace(kv->kv_value); + value_size = temp_data_buffer->size(); + return true; + } else { + temp_data_buffer.reset(); + value_size = 0; + return false; + } + } + + uint32_t kv_get_data(uint32_t offset, char* data, uint32_t data_size) override { + const char* temp = nullptr; + uint32_t temp_size = 0; + if (temp_data_buffer) { + temp = temp_data_buffer->data(); + temp_size = temp_data_buffer->size(); + } + if (offset < temp_size) + memcpy(data, temp + offset, std::min(data_size, temp_size - offset)); + return temp_size; + } + + std::unique_ptr kv_it_create(uint64_t contract, const char* prefix, uint32_t size) override { + EOS_ASSERT(num_iterators < limits.max_iterators, kv_bad_iter, "Too many iterators"); + EOS_ASSERT(size <= limits.max_key_size, kv_bad_iter, "Prefix too large"); + return std::make_unique(db, tracker, num_iterators, name{ contract }, + std::vector{ prefix, prefix + size }); + } + }; // kv_context_chainbase + + template + std::unique_ptr create_kv_chainbase_context(chainbase::database& db, name receiver, + Resource_manager resource_manager, const kv_database_config& limits) + { + return std::make_unique>(db,receiver, resource_manager, limits); + } + +}} // namespace eosio::chain diff --git a/libraries/chain/include/eosio/chain/backing_store/kv_context_rocksdb.hpp b/libraries/chain/include/eosio/chain/backing_store/kv_context_rocksdb.hpp new file mode 100644 index 00000000000..9b2c290ef9a --- /dev/null +++ b/libraries/chain/include/eosio/chain/backing_store/kv_context_rocksdb.hpp @@ -0,0 +1,496 @@ +#include +#include +#include +#include +#include +#include + +#include + +namespace eosio { namespace chain { + + static inline uint32_t actual_key_size(const uint32_t raw_key_size) { + static auto rocks_prefix = make_rocksdb_contract_kv_prefix(); + static auto prefix_size = rocks_prefix.size() + sizeof(uint64_t); // kv.db prefix + contract size. + EOS_ASSERT(raw_key_size >= prefix_size, kv_rocksdb_bad_value_size_exception, + "The size of key returned from RocksDB is less than prefix size"); + return raw_key_size - prefix_size; + } + + + static inline const char* actual_key_start(const char* key) { + static auto rocks_prefix = make_rocksdb_contract_kv_prefix(); + static auto prefix_size = rocks_prefix.size() + sizeof(uint64_t); + return key + prefix_size; + } + + // Need to store payer so that this account is properly + // credited when storage is removed or changed + // to another payer + inline static eosio::session::shared_bytes build_value(const char* value, uint32_t value_size, const account_name& payer) { + const uint32_t final_value_size = backing_store::payer_in_value_size + value_size; + auto result = eosio::session::shared_bytes(final_value_size); + + char buf[backing_store::payer_in_value_size]; + memcpy(buf, &payer, backing_store::payer_in_value_size); + std::memcpy(result.data(), buf, backing_store::payer_in_value_size); + std::memcpy(result.data() + backing_store::payer_in_value_size, value, value_size); + return result; + } + + + static inline eosio::session::shared_bytes make_prefix_key(uint64_t contract, const char* user_prefix, uint32_t user_prefix_size) { + static auto rocks_prefix = make_rocksdb_contract_kv_prefix(); + + auto result = eosio::session::shared_bytes(rocks_prefix.size() + sizeof(contract) + user_prefix_size); + std::memcpy(result.data(), rocks_prefix.data(), rocks_prefix.size()); + b1::chain_kv::insert_key(result, rocks_prefix.size(), contract); + std::memcpy(result.data() + rocks_prefix.size() + sizeof(contract), user_prefix, user_prefix_size); + return result; + } + + static inline eosio::session::shared_bytes + make_composite_key(uint64_t contract, const char* user_prefix, uint32_t user_prefix_size, const char* key, uint32_t key_size) { + static auto rocks_prefix = make_rocksdb_contract_kv_prefix(); + + auto result = eosio::session::shared_bytes(rocks_prefix.size() + sizeof(contract) + user_prefix_size + key_size); + std::memcpy(result.data(), rocks_prefix.data(), rocks_prefix.size()); + b1::chain_kv::insert_key(result, rocks_prefix.size(), contract); + std::memcpy(result.data() + rocks_prefix.size() + sizeof(contract), user_prefix, user_prefix_size); + std::memcpy(result.data() + rocks_prefix.size() + sizeof(contract) + user_prefix_size, key, key_size); + return result; + } + + static inline eosio::session::shared_bytes make_composite_key(const eosio::session::shared_bytes& prefix, + const char* key, uint32_t key_size) { + auto result = eosio::session::shared_bytes(prefix.size() + key_size); + std::memcpy(result.data(), prefix.data(), prefix.size()); + std::memcpy(result.data() + prefix.size(), key, key_size); + return result; + } + + static inline int32_t compare_bytes(const eosio::session::shared_bytes& left, const eosio::session::shared_bytes& right) { + if (left < right) { + return -1; + } + + if (left > right) { + return 1; + } + + return 0; + }; + + template + struct kv_iterator_rocksdb : kv_iterator { + using session_type = std::remove_pointer_t; + + uint32_t& num_iterators; + uint64_t kv_contract{0}; + eosio::session::shared_bytes kv_prefix; // Format: [contract, prefix] + session_type* kv_session{nullptr}; + typename session_type::iterator kv_begin; + typename session_type::iterator kv_end; + typename session_type::iterator kv_current; + + kv_iterator_rocksdb(uint32_t& num_iterators, session_type& session, uint64_t contract, const char* user_prefix, + uint32_t user_prefix_size) + : num_iterators{ num_iterators }, kv_contract{ contract }, + kv_prefix{ make_prefix_key(contract, user_prefix, user_prefix_size) }, + kv_session{ &session }, kv_begin{ kv_session->lower_bound(kv_prefix) }, + kv_end{ [&](){ auto kv_next_prefix = kv_prefix.next(); return kv_session->lower_bound(kv_next_prefix); }() }, + kv_current{ kv_end } { + ++num_iterators; + } + + ~kv_iterator_rocksdb() override { --num_iterators; } + + bool is_kv_chainbase_context_iterator() const override { return false; } + bool is_kv_rocksdb_context_iterator() const override { return true; } + + kv_it_stat kv_it_status() override { + if (kv_current == kv_end) + return kv_it_stat::iterator_end; + else if (kv_current.deleted()) + return kv_it_stat::iterator_erased; + else + return kv_it_stat::iterator_ok; + } + + int32_t kv_it_compare(const kv_iterator& rhs) override { + EOS_ASSERT(rhs.is_kv_rocksdb_context_iterator(), kv_bad_iter, "Incompatible key-value iterators"); + auto& r = const_cast(static_cast(rhs)); + EOS_ASSERT(kv_session == r.kv_session && kv_prefix == r.kv_prefix, kv_bad_iter, + "Incompatible key-value iterators"); + EOS_ASSERT(!kv_current.deleted(), kv_bad_iter, "Iterator to erased element"); + EOS_ASSERT(!r.kv_current.deleted(), kv_bad_iter, "Iterator to erased element"); + try { + try { + auto left_status = kv_it_status(); + auto right_status = r.kv_it_status(); + + if (left_status == kv_it_stat::iterator_end && right_status == kv_it_stat::iterator_end) { + return 0; + } + if (left_status == kv_it_stat::iterator_end) { + return 1; + } + if (right_status == kv_it_stat::iterator_end) { + return -1; + } + + auto result = compare_bytes(kv_current.key(), r.kv_current.key()); + if (result) { + return result; + } + auto left = (*kv_current).second; + auto right = (*r.kv_current).second; + if (left && right) { + return compare_bytes(*left, *right); + } + return left.has_value() == right.has_value(); + } + FC_LOG_AND_RETHROW() + } + CATCH_AND_EXIT_DB_FAILURE() + } + + int32_t kv_it_key_compare(const char* key, uint32_t size) override { + EOS_ASSERT(!kv_current.deleted(), kv_bad_iter, "Iterator to erased element"); + try { + try { + auto current_key = eosio::session::shared_bytes{}; + if (kv_it_status() == kv_it_stat::iterator_ok) { + current_key = kv_current.key(); + } else { + return 1; + } + return compare_bytes(current_key, make_composite_key(kv_contract, nullptr, 0, key, size)); + } + FC_LOG_AND_RETHROW() + } + CATCH_AND_EXIT_DB_FAILURE() + } + + kv_it_stat kv_it_move_to_end() override { + try { + try { + kv_current = kv_end; + return kv_it_stat::iterator_end; + } + FC_LOG_AND_RETHROW() + } + CATCH_AND_EXIT_DB_FAILURE() + } + + kv_it_stat kv_it_next(uint32_t* found_key_size, uint32_t* found_value_size) override { + EOS_ASSERT(!kv_current.deleted(), kv_bad_iter, "Iterator to erased element"); + kv_it_stat status; + try { + try { + if (kv_current == kv_end) { + kv_current = kv_begin; + } else { + ++kv_current; + } + status = get_current_key_value_sizes(found_key_size, found_value_size); + } + FC_LOG_AND_RETHROW() + } + CATCH_AND_EXIT_DB_FAILURE() + + return status; + } + + kv_it_stat kv_it_prev(uint32_t* found_key_size, uint32_t* found_value_size) override { + EOS_ASSERT(!kv_current.deleted(), kv_bad_iter, "Iterator to erased element"); + kv_it_stat status; + try { + try { + if (kv_current == kv_begin) { + kv_current = kv_end; + } else { + --kv_current; + } + status = get_current_key_value_sizes(found_key_size, found_value_size); + } + FC_LOG_AND_RETHROW() + } + CATCH_AND_EXIT_DB_FAILURE() + + return status; + } + + kv_it_stat kv_it_lower_bound(const char* key, uint32_t size, uint32_t* found_key_size, + uint32_t* found_value_size) override { + kv_it_stat status; + try { + try { + auto key_bytes = make_composite_key(kv_contract, nullptr, 0, key, size); + if (key_bytes < kv_prefix) { + key_bytes = kv_prefix; + } + + kv_current = kv_session->lower_bound(key_bytes); + status = get_current_key_value_sizes(found_key_size, found_value_size); + } + FC_LOG_AND_RETHROW() + } + CATCH_AND_EXIT_DB_FAILURE() + + return status; + } + + kv_it_stat kv_it_key(uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) override { + EOS_ASSERT(!kv_current.deleted(), kv_bad_iter, "Iterator to erased element"); + + auto key = eosio::session::shared_bytes{}; + try { + try { + if (kv_it_status() == kv_it_stat::iterator_ok) { + key = kv_current.key(); + } + } + FC_LOG_AND_RETHROW() + } + CATCH_AND_EXIT_DB_FAILURE() + + if (key) { + actual_size = actual_key_size(key.size()); + if (offset < actual_size) { + std::memcpy(dest, actual_key_start(key.data()) + offset, std::min(size, actual_size - offset)); + } + return kv_it_stat::iterator_ok; + } else { + actual_size = 0; + return kv_it_stat::iterator_end; + } + } + + kv_it_stat kv_it_value(uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) override { + EOS_ASSERT(!kv_current.deleted(), kv_bad_iter, "Iterator to erased element"); + + auto kv = std::pair{ eosio::session::shared_bytes{}, std::optional{} }; + try { + try { + if (kv_it_status() == kv_it_stat::iterator_ok) { + kv = *kv_current; + } + } + FC_LOG_AND_RETHROW() + } + CATCH_AND_EXIT_DB_FAILURE() + + auto& value = kv.second; + if (value) { + actual_size = backing_store::actual_value_size(value->size()); + if (offset < actual_size) { + std::memcpy(dest, backing_store::actual_value_start(value->data()) + offset, std::min(size, actual_size - offset)); + } + return kv_it_stat::iterator_ok; + } else { + actual_size = 0; + return kv_it_stat::iterator_end; + } + } + + std::optional kv_it_payer() override { + EOS_ASSERT(!kv_current.deleted(), kv_bad_iter, "Iterator to erased element"); + if (kv_it_status() == kv_it_stat::iterator_ok) { + const auto& value = (*kv_current).second; + if (value) { + return backing_store::payer_payload(*value).payer; + } + } + return {}; + } + + private: + kv_it_stat get_current_key_value_sizes(uint32_t* found_key_size, uint32_t* found_value_size) { + auto status = kv_it_status(); + if (status == kv_it_stat::iterator_ok) { + // Not end or at an erased kv pair + auto kv = *kv_current; + *found_value_size = backing_store::actual_value_size( + kv.second->size()); // This must be before *found_key_size in case actual_value_size throws + *found_key_size = actual_key_size(kv.first.size()); + } else { + *found_key_size = 0; + *found_value_size = 0; + } + return status; + } + }; // kv_iterator_rocksdb + + template + struct kv_context_rocksdb : kv_context { + using session_type = std::remove_pointer_t; + + session_type* session; + name receiver; + Resource_manager resource_manager; + const kv_database_config& limits; + uint32_t num_iterators = 0; + eosio::session::shared_bytes current_key; + std::optional current_value; + + kv_context_rocksdb(session_type& the_session, name receiver, Resource_manager /*kv_resource_manager*/ resource_manager, + const kv_database_config& limits) + : session{ &the_session }, receiver{ receiver }, + resource_manager{ resource_manager }, limits{ limits } {} + + int64_t kv_erase(uint64_t contract, const char* key, uint32_t key_size) override { + const name contract_name {contract}; + EOS_ASSERT(contract_name == receiver, table_operation_not_permitted, "Can not write to this key"); + + current_value.reset(); + current_key = eosio::session::shared_bytes{}; + auto old_value = std::optional{}; + std::optional pp; + try { + try { + auto composite_key = make_composite_key(contract, nullptr, 0, key, key_size); + const auto old_value = session->read(composite_key); + if (!old_value) + return 0; + pp = backing_store::payer_payload(*old_value); + session->erase(composite_key); + } + FC_LOG_AND_RETHROW() + } + CATCH_AND_EXIT_DB_FAILURE() + + const int64_t resource_delta = erase_table_usage(resource_manager, pp->payer, key, key_size, pp->value_size); + + if (auto dm_logger = resource_manager._context->control.get_deep_mind_logger()) { + fc_dlog(*dm_logger, "KV_OP REM ${action_id} ${db} ${payer} ${key} ${odata}", + ("action_id", resource_manager._context->get_action_id()) + ("contract", contract_name) + ("payer", pp->payer) + ("key", fc::to_hex(key, key_size)) + ("odata", fc::to_hex(pp->value, pp->value_size)) + ); + } + return resource_delta; + } + + int64_t kv_set(uint64_t contract, const char* key, uint32_t key_size, const char* value, uint32_t value_size, + account_name payer) override { + const name contract_name {contract}; + EOS_ASSERT(contract_name == receiver, table_operation_not_permitted, "Can not write to this key"); + EOS_ASSERT(key_size <= limits.max_key_size, kv_limit_exceeded, "Key too large"); + EOS_ASSERT(value_size <= limits.max_value_size, kv_limit_exceeded, "Value too large"); + + current_value.reset(); + current_key = eosio::session::shared_bytes{}; + auto old_value = std::optional{}; + std::optional old_pp; + try { + try { + auto composite_key = make_composite_key(contract, nullptr, 0, key, key_size); + old_value = session->read(composite_key); + if (old_value) { + old_pp = backing_store::payer_payload(*old_value); + } + + backing_store::payer_payload new_value (payer, value, value_size); + session->write(composite_key, new_value.as_payload()); + } + FC_LOG_AND_RETHROW() + } + CATCH_AND_EXIT_DB_FAILURE() + + auto resource_delta = int64_t{ 0 }; + if (old_value) { + resource_delta = + update_table_usage(resource_manager, old_pp->payer, payer, key, key_size, old_pp->value_size, value_size); + + if (auto dm_logger = resource_manager._context->control.get_deep_mind_logger()) { + fc_dlog(*dm_logger, "KV_OP UPD ${action_id} ${db} ${payer} ${key} ${odata}:${ndata}", + ("action_id", resource_manager._context->get_action_id()) + ("contract", contract_name) + ("payer", payer) + ("key", fc::to_hex(key, key_size)) + ("odata", fc::to_hex(old_pp->value, old_pp->value_size)) + ("ndata", fc::to_hex(value, value_size)) + ); + } + } else { + resource_delta = create_table_usage(resource_manager, payer, key, key_size, value_size); + + if (auto dm_logger = resource_manager._context->control.get_deep_mind_logger()) { + fc_dlog(*dm_logger, "KV_OP INS ${action_id} ${db} ${payer} ${key} ${ndata}", + ("action_id", resource_manager._context->get_action_id()) + ("contract", contract_name) + ("payer", payer) + ("key", fc::to_hex(key, key_size)) + ("ndata", fc::to_hex(value, value_size)) + ); + } + } + + return resource_delta; + } + + bool kv_get(uint64_t contract, const char* key, uint32_t key_size, uint32_t& value_size) override { + current_key = eosio::session::shared_bytes{}; + current_value.reset(); + try { + try { + current_key = make_composite_key(contract, nullptr, 0, key, key_size); + current_value = session->read(current_key); + } + FC_LOG_AND_RETHROW() + } + CATCH_AND_EXIT_DB_FAILURE() + + if (current_value) { + value_size = backing_store::actual_value_size(current_value->size()); + return true; + } else { + value_size = 0; + return false; + } + } + + uint32_t kv_get_data(uint32_t offset, char* data, uint32_t data_size) override { + uint32_t temp_size = 0; + if (current_value) { + temp_size = backing_store::actual_value_size(current_value->size()); + } + if (offset < temp_size) { + const char* temp = backing_store::actual_value_start(current_value->data()); + std::memcpy(data, temp + offset, std::min(data_size, temp_size - offset)); + } + return temp_size; + } + + std::unique_ptr kv_it_create(uint64_t contract, const char* prefix, uint32_t size) override { + EOS_ASSERT(num_iterators < limits.max_iterators, kv_bad_iter, "Too many iterators"); + EOS_ASSERT(size <= limits.max_key_size, kv_bad_iter, "Prefix too large"); + try { + try { + return std::make_unique>(num_iterators, *session, contract, prefix, + size); + } + FC_LOG_AND_RETHROW() + } + CATCH_AND_EXIT_DB_FAILURE() + } + }; // kv_context_rocksdb + + template + std::unique_ptr create_kv_rocksdb_context(std::remove_pointer_t& session, name receiver, + Resource_manager resource_manager, + const kv_database_config& limits) { + try { + try { + return std::make_unique>(session, receiver, + resource_manager, limits); + } + FC_LOG_AND_RETHROW() + } + CATCH_AND_EXIT_DB_FAILURE() + } + +}} // namespace eosio::chain diff --git a/libraries/chain/include/eosio/chain/block.hpp b/libraries/chain/include/eosio/chain/block.hpp index 6229c6760c0..ed08822babd 100644 --- a/libraries/chain/include/eosio/chain/block.hpp +++ b/libraries/chain/include/eosio/chain/block.hpp @@ -30,23 +30,24 @@ namespace eosio { namespace chain { fc::unsigned_int net_usage_words; ///< total billed NET usage, so we can reconstruct resource state when skipping context free data... hard failures... }; - struct transaction_receipt : public transaction_receipt_header { + struct transaction_receipt_v0 : public transaction_receipt_header { + using trx_type = std::variant; + transaction_receipt_v0() : transaction_receipt_header() {} + transaction_receipt_v0(const transaction_receipt_header& header, trx_type&& t): transaction_receipt_header(header), trx(std::move(t)){} + explicit transaction_receipt_v0( transaction_id_type tid ):transaction_receipt_header(executed),trx(std::move(tid)){} + explicit transaction_receipt_v0( packed_transaction_v0 ptrx ):transaction_receipt_header(executed),trx(std::move(ptrx)){} - transaction_receipt():transaction_receipt_header(){} - explicit transaction_receipt( const transaction_id_type& tid ):transaction_receipt_header(executed),trx(tid){} - explicit transaction_receipt( const packed_transaction& ptrx ):transaction_receipt_header(executed),trx(ptrx){} - - fc::static_variant trx; + trx_type trx; digest_type digest()const { digest_type::encoder enc; fc::raw::pack( enc, status ); fc::raw::pack( enc, cpu_usage_us ); fc::raw::pack( enc, net_usage_words ); - if( trx.contains() ) - fc::raw::pack( enc, trx.get() ); + if( std::holds_alternative(trx) ) + fc::raw::pack( enc, std::get(trx) ); else - fc::raw::pack( enc, trx.get().packed_digest() ); + fc::raw::pack( enc, std::get(trx).packed_digest() ); return enc.result(); } }; @@ -73,7 +74,7 @@ namespace eosio { namespace chain { namespace detail { template struct block_extension_types { - using block_extension_t = fc::static_variant< Ts... >; + using block_extension_t = std::variant< Ts... >; using decompose_t = decompose< Ts... >; }; } @@ -86,18 +87,86 @@ namespace eosio { namespace chain { /** */ + struct signed_block_v0 : public signed_block_header{ + private: + signed_block_v0( const signed_block_v0& ) = default; + public: + signed_block_v0() = default; + explicit signed_block_v0( const signed_block_header& h ):signed_block_header(h){} + signed_block_v0( signed_block_v0&& ) = default; + signed_block_v0& operator=(signed_block_v0&&) = default; + signed_block_v0& operator=(const signed_block_v0&) = delete; + signed_block_v0 clone() const { return *this; } + + deque transactions; /// new or generated transactions + extensions_type block_extensions; + + flat_multimap validate_and_extract_extensions()const; + }; + + using signed_block_v0_ptr = std::shared_ptr; + + struct transaction_receipt : public transaction_receipt_header { + + transaction_receipt():transaction_receipt_header(){} + transaction_receipt(const transaction_receipt_v0&, bool legacy); + transaction_receipt(transaction_receipt_v0&&, bool legacy); + explicit transaction_receipt( const transaction_id_type& tid ):transaction_receipt_header(executed),trx(tid){} + + explicit transaction_receipt( const packed_transaction& ptrx ):transaction_receipt_header(executed),trx(std::in_place_type, ptrx){} + + std::variant trx; + + std::size_t maximum_pruned_pack_size( packed_transaction::cf_compression_type segment_compression ) const; + + digest_type digest()const { + digest_type::encoder enc; + fc::raw::pack( enc, status ); + fc::raw::pack( enc, cpu_usage_us ); + fc::raw::pack( enc, net_usage_words ); + if( std::holds_alternative(trx) ) + fc::raw::pack( enc, std::get(trx) ); + else + fc::raw::pack( enc, std::get(trx).packed_digest() ); + return enc.result(); + } + }; + struct signed_block : public signed_block_header{ private: signed_block( const signed_block& ) = default; public: + enum class prune_state_type : uint8_t { incomplete, complete, complete_legacy }; + signed_block() = default; explicit signed_block( const signed_block_header& h ):signed_block_header(h){} + signed_block( const signed_block_v0&, bool legacy ); + signed_block( signed_block_v0&&, bool legacy ); signed_block( signed_block&& ) = default; signed_block& operator=(const signed_block&) = delete; + signed_block& operator=(signed_block&&) = default; signed_block clone() const { return *this; } - - vector transactions; /// new or generated transactions - extensions_type block_extensions; + std::optional to_signed_block_v0() const; + + fc::enum_type prune_state{prune_state_type::complete_legacy}; + deque transactions; /// new or generated transactions + extensions_type block_extensions; + + std::size_t maximum_pruned_pack_size( packed_transaction::cf_compression_type segment_compression ) const; + + // Returns the maximum_pruned_padded_size. It is the caller's responsibility to + // reserve enough space after the end if in-place pruning is desired. + template + std::size_t pack(Stream& stream, packed_transaction::cf_compression_type segment_compression) const { + std::size_t padded_size = maximum_pruned_pack_size( segment_compression ); + // TODO: This only handles legacy transactions. + fc::raw::pack(stream, *this); + return padded_size; + } + template + void unpack(Stream& stream, packed_transaction::cf_compression_type segment_compression) { + fc::raw::unpack(stream, *this); + } flat_multimap validate_and_extract_extensions()const; }; @@ -112,10 +181,15 @@ namespace eosio { namespace chain { } } /// eosio::chain -FC_REFLECT_ENUM( eosio::chain::transaction_receipt::status_enum, +FC_REFLECT_ENUM( eosio::chain::transaction_receipt_header::status_enum, (executed)(soft_fail)(hard_fail)(delayed)(expired) ) +FC_REFLECT_ENUM( eosio::chain::signed_block::prune_state_type, + (incomplete)(complete)(complete_legacy) ) FC_REFLECT(eosio::chain::transaction_receipt_header, (status)(cpu_usage_us)(net_usage_words) ) -FC_REFLECT_DERIVED(eosio::chain::transaction_receipt, (eosio::chain::transaction_receipt_header), (trx) ) +FC_REFLECT_DERIVED(eosio::chain::transaction_receipt_v0, (eosio::chain::transaction_receipt_header), (trx) ) FC_REFLECT(eosio::chain::additional_block_signatures_extension, (signatures)); -FC_REFLECT_DERIVED(eosio::chain::signed_block, (eosio::chain::signed_block_header), (transactions)(block_extensions) ) +FC_REFLECT_DERIVED(eosio::chain::signed_block_v0, (eosio::chain::signed_block_header), (transactions)(block_extensions) ) + +FC_REFLECT_DERIVED(eosio::chain::transaction_receipt, (eosio::chain::transaction_receipt_header), (trx) ) +FC_REFLECT_DERIVED(eosio::chain::signed_block, (eosio::chain::signed_block_header), (prune_state)(transactions)(block_extensions) ) diff --git a/libraries/chain/include/eosio/chain/block_header.hpp b/libraries/chain/include/eosio/chain/block_header.hpp index 286ecf528c1..001dc2d4006 100644 --- a/libraries/chain/include/eosio/chain/block_header.hpp +++ b/libraries/chain/include/eosio/chain/block_header.hpp @@ -10,7 +10,7 @@ namespace eosio { namespace chain { namespace detail { template struct block_header_extension_types { - using block_header_extension_t = fc::static_variant< Ts... >; + using block_header_extension_t = std::variant< Ts... >; using decompose_t = decompose< Ts... >; }; } @@ -53,7 +53,7 @@ namespace eosio { namespace chain { * irreversible and that it the new producer schedule takes effect this block. */ - using new_producers_type = optional; + using new_producers_type = std::optional; uint32_t schedule_version = 0; new_producers_type new_producers; @@ -63,7 +63,7 @@ namespace eosio { namespace chain { block_header() = default; digest_type digest()const; - block_id_type id() const; + block_id_type calculate_id() const; uint32_t block_num() const { return num_from_id(previous) + 1; } static uint32_t num_from_id(const block_id_type& id); @@ -75,7 +75,7 @@ namespace eosio { namespace chain { { signature_type producer_signature; }; - + } } /// namespace eosio::chain FC_REFLECT(eosio::chain::block_header, diff --git a/libraries/chain/include/eosio/chain/block_header_state.hpp b/libraries/chain/include/eosio/chain/block_header_state.hpp index 956d66ba422..259a593fa21 100644 --- a/libraries/chain/include/eosio/chain/block_header_state.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state.hpp @@ -83,7 +83,7 @@ struct pending_block_header_state : public detail::block_header_state_common { signed_block_header make_block_header( const checksum256_type& transaction_mroot, const checksum256_type& action_mroot, - const optional& new_producers, + const std::optional& new_producers, vector&& new_protocol_feature_activations, const protocol_feature_set& pfs)const; @@ -177,6 +177,7 @@ FC_REFLECT( eosio::chain::detail::schedule_info, (schedule) ) +// @ignore header_exts FC_REFLECT_DERIVED( eosio::chain::block_header_state, (eosio::chain::detail::block_header_state_common), (id) (header) diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp index 5fbe3e771a5..7858df0fe50 100644 --- a/libraries/chain/include/eosio/chain/block_log.hpp +++ b/libraries/chain/include/eosio/chain/block_log.hpp @@ -2,19 +2,21 @@ #include #include #include +#include +#include namespace eosio { namespace chain { namespace detail { class block_log_impl; } /* The block log is an external append only log of the blocks with a header. Blocks should only - * be written to the log after they irreverisble as the log is append only. The log is a doubly + * be written to the log after they are irreversible as the log is append only. The log is a doubly * linked list of blocks. There is a secondary index file of only block positions that enables * O(1) random access lookup by block number. * - * +---------+----------------+---------+----------------+-----+------------+-------------------+ - * | Block 1 | Pos of Block 1 | Block 2 | Pos of Block 2 | ... | Head Block | Pos of Head Block | - * +---------+----------------+---------+----------------+-----+------------+-------------------+ + * +---------------+----------------+---------------+----------------+-----+------------------+-------------------+ + * | Block 1 Entry | Pos of Block 1 | Block 2 Entry | Pos of Block 2 | ... | Head Block Entry | Pos of Head Block | + * +---------------+----------------+---------------+----------------+-----+------------------+-------------------+ * * +----------------+----------------+-----+-------------------+ * | Pos of Block 1 | Pos of Block 2 | ... | Pos of Head Block | @@ -32,42 +34,52 @@ namespace eosio { namespace chain { * linear scan of the main file. */ + namespace bfs = boost::filesystem; + class block_log { public: - block_log(const fc::path& data_dir); - block_log(block_log&& other); + + using config_type = block_log_config; + + block_log(const config_type& config); + block_log(block_log&& other) = default; ~block_log(); + + uint64_t append(const signed_block_ptr& block, packed_transaction::cf_compression_type segment_compression); - uint64_t append(const signed_block_ptr& b); - void flush(); - void reset( const genesis_state& gs, const signed_block_ptr& genesis_block ); - void reset( const chain_id_type& chain_id, uint32_t first_block_num ); + // create futures for append, must call in order of blocks + std::future>> + create_append_future(boost::asio::io_context& thread_pool, + const signed_block_ptr& b, packed_transaction::cf_compression_type segment_compression); + uint64_t append(std::future>> f); - signed_block_ptr read_block(uint64_t file_pos)const; - void read_block_header(block_header& bh, uint64_t file_pos)const; - signed_block_ptr read_block_by_num(uint32_t block_num)const; + void reset( const genesis_state& gs, const signed_block_ptr& genesis_block, packed_transaction::cf_compression_type segment_compression); + void reset( const chain_id_type& chain_id, uint32_t first_block_num ); + block_id_type read_block_id_by_num(uint32_t block_num)const; - signed_block_ptr read_block_by_id(const block_id_type& id)const { - return read_block_by_num(block_header::num_from_id(id)); - } + std::unique_ptr read_signed_block_by_num(uint32_t block_num) const; + + const signed_block_ptr& head() const; + uint32_t first_block_num() const; + + static bool exists(const fc::path& data_dir); /** - * Return offset of block in file, or block_log::npos if it does not exist. - */ - uint64_t get_block_pos(uint32_t block_num) const; - signed_block_ptr read_head()const; - const signed_block_ptr& head()const; - const block_id_type& head_id()const; - uint32_t first_block_num() const; + * @param ids[in,out] The list of transaction ids to be pruned. After the member function returns, + * it would be modified to contain the list of transaction ids that do not + * exists in the specified block. + * @returns The number of transactions been pruned + **/ + size_t prune_transactions(uint32_t block_num, vector& ids); static const uint64_t npos = std::numeric_limits::max(); static const uint32_t min_supported_version; static const uint32_t max_supported_version; - static fc::path repair_log( const fc::path& data_dir, uint32_t truncate_at_block = 0 ); + static fc::path repair_log( const fc::path& data_dir, uint32_t truncate_at_block = UINT32_MAX, const char* reversible_block_dir_name="" ); - static fc::optional extract_genesis_state( const fc::path& data_dir ); + static std::optional extract_genesis_state( const fc::path& data_dir ); static chain_id_type extract_chain_id( const fc::path& data_dir ); @@ -80,35 +92,18 @@ namespace eosio { namespace chain { static bool is_supported_version(uint32_t version); static bool trim_blocklog_front(const fc::path& block_dir, const fc::path& temp_dir, uint32_t truncate_at_block); + static int trim_blocklog_end(fc::path block_dir, uint32_t n); - private: - void open(const fc::path& data_dir); - void construct_index(); + // used for unit test to generate older version blocklog + static void set_version(uint32_t); + uint32_t version() const; - std::unique_ptr my; - }; + /** + * @param n Only test 1 block out of every n blocks. If n is 0, it is maximum between 1 and the ceiling of the total number blocks divided by 8. + */ + static void smoke_test(fc::path block_dir, uint32_t n); -//to derive blknum_offset==14 see block_header.hpp and note on disk struct is packed -// block_timestamp_type timestamp; //bytes 0:3 -// account_name producer; //bytes 4:11 -// uint16_t confirmed; //bytes 12:13 -// block_id_type previous; //bytes 14:45, low 4 bytes is big endian block number of previous block - - struct trim_data { //used by trim_blocklog_front(), trim_blocklog_end(), and smoke_test() - trim_data(fc::path block_dir); - ~trim_data(); - uint64_t block_index(uint32_t n) const; - uint64_t block_pos(uint32_t n); - fc::path block_file_name, index_file_name; //full pathname for blocks.log and blocks.index - uint32_t version = 0; //blocklog version - uint32_t first_block = 0; //first block in blocks.log - uint32_t last_block = 0; //last block in blocks.log - FILE* blk_in = nullptr; //C style files for reading blocks.log and blocks.index - FILE* ind_in = nullptr; //C style files for reading blocks.log and blocks.index - //we use low level file IO because it is distinctly faster than C++ filebuf or iostream - uint64_t first_block_pos = 0; //file position in blocks.log for the first block in the log - chain_id_type chain_id; - - static constexpr int blknum_offset{14}; //offset from start of block to 4 byte block number, valid for the only allowed versions + private: + std::unique_ptr my; }; } } diff --git a/libraries/chain/include/eosio/chain/block_log_config.hpp b/libraries/chain/include/eosio/chain/block_log_config.hpp new file mode 100644 index 00000000000..5e3401bc61d --- /dev/null +++ b/libraries/chain/include/eosio/chain/block_log_config.hpp @@ -0,0 +1,19 @@ +#pragma once +#include + +namespace eosio { +namespace chain { + +namespace bfs = boost::filesystem; + +struct block_log_config { + bfs::path log_dir; + bfs::path retained_dir; + bfs::path archive_dir; + uint32_t stride = UINT32_MAX; + uint16_t max_retained_files = 10; + bool fix_irreversible_blocks = false; +}; + +} // namespace chain +} // namespace eosio diff --git a/libraries/chain/include/eosio/chain/block_state.hpp b/libraries/chain/include/eosio/chain/block_state.hpp index 6385500e5d2..3ebafe87a76 100644 --- a/libraries/chain/include/eosio/chain/block_state.hpp +++ b/libraries/chain/include/eosio/chain/block_state.hpp @@ -19,7 +19,7 @@ namespace eosio { namespace chain { block_state( pending_block_header_state&& cur, signed_block_ptr&& b, // unsigned block - vector&& trx_metas, + deque&& trx_metas, const protocol_feature_set& pfs, const std::function&, @@ -44,24 +44,24 @@ namespace eosio { namespace chain { bool is_valid()const { return validated; } bool is_pub_keys_recovered()const { return _pub_keys_recovered; } - vector extract_trxs_metas() { + deque extract_trxs_metas() { _pub_keys_recovered = false; auto result = std::move( _cached_trxs ); _cached_trxs.clear(); return result; } - void set_trxs_metas( vector&& trxs_metas, bool keys_recovered ) { + void set_trxs_metas( deque&& trxs_metas, bool keys_recovered ) { _pub_keys_recovered = keys_recovered; _cached_trxs = std::move( trxs_metas ); } - const vector& trxs_metas()const { return _cached_trxs; } + const deque& trxs_metas()const { return _cached_trxs; } bool validated = false; bool _pub_keys_recovered = false; /// this data is redundant with the data stored in block, but facilitates /// recapturing transactions when we pop a block - vector _cached_trxs; + deque _cached_trxs; }; using block_state_ptr = std::shared_ptr; @@ -69,4 +69,5 @@ namespace eosio { namespace chain { } } /// namespace eosio::chain +// @ignore _pub_keys_recovered _cached_trxs FC_REFLECT_DERIVED( eosio::chain::block_state, (eosio::chain::block_header_state), (block)(validated) ) diff --git a/libraries/chain/include/eosio/chain/block_timestamp.hpp b/libraries/chain/include/eosio/chain/block_timestamp.hpp index 821e44efbff..a20f609ddcc 100644 --- a/libraries/chain/include/eosio/chain/block_timestamp.hpp +++ b/libraries/chain/include/eosio/chain/block_timestamp.hpp @@ -5,7 +5,6 @@ #include #include #include -#include #include namespace eosio { namespace chain { diff --git a/libraries/chain/include/eosio/chain/chain_config.hpp b/libraries/chain/include/eosio/chain/chain_config.hpp index 9e83b09779f..7a0349fe6ee 100644 --- a/libraries/chain/include/eosio/chain/chain_config.hpp +++ b/libraries/chain/include/eosio/chain/chain_config.hpp @@ -2,6 +2,7 @@ #include #include +#include namespace eosio { namespace chain { @@ -12,7 +13,30 @@ namespace eosio { namespace chain { * their preference for each of the parameters in this object, and the blockchain runs according to the median of the * values specified by the producers. */ -struct chain_config { +struct chain_config_v0 { + + //order must match parameters as ids are used in serialization + enum { + max_block_net_usage_id, + target_block_net_usage_pct_id, + max_transaction_net_usage_id, + base_per_transaction_net_usage_id, + net_usage_leeway_id, + context_free_discount_net_usage_num_id, + context_free_discount_net_usage_den_id, + max_block_cpu_usage_id, + target_block_cpu_usage_pct_id, + max_transaction_cpu_usage_id, + min_transaction_cpu_usage_id, + max_transaction_lifetime_id, + deferred_trx_expiration_window_id, + max_transaction_delay_id, + max_inline_action_size_id, + max_inline_action_depth_id, + max_authority_depth_id, + PARAMS_COUNT + }; + uint64_t max_block_net_usage; ///< the maxiumum net usage in instructions for a block uint32_t target_block_net_usage_pct; ///< the target percent (1% == 100, 100%= 10,000) of maximum net usage; exceeding this triggers congestion handling uint32_t max_transaction_net_usage; ///< the maximum objectively measured net usage that the chain will allow regardless of account limits @@ -35,29 +59,16 @@ struct chain_config { void validate()const; + inline const chain_config_v0& v0() const { + return *this; + } + template - friend Stream& operator << ( Stream& out, const chain_config& c ) { - return out << "Max Block Net Usage: " << c.max_block_net_usage << ", " - << "Target Block Net Usage Percent: " << ((double)c.target_block_net_usage_pct / (double)config::percent_1) << "%, " - << "Max Transaction Net Usage: " << c.max_transaction_net_usage << ", " - << "Base Per-Transaction Net Usage: " << c.base_per_transaction_net_usage << ", " - << "Net Usage Leeway: " << c.net_usage_leeway << ", " - << "Context-Free Data Net Usage Discount: " << (double)c.context_free_discount_net_usage_num * 100.0 / (double)c.context_free_discount_net_usage_den << "% , " - - << "Max Block CPU Usage: " << c.max_block_cpu_usage << ", " - << "Target Block CPU Usage Percent: " << ((double)c.target_block_cpu_usage_pct / (double)config::percent_1) << "%, " - << "Max Transaction CPU Usage: " << c.max_transaction_cpu_usage << ", " - << "Min Transaction CPU Usage: " << c.min_transaction_cpu_usage << ", " - - << "Max Transaction Lifetime: " << c.max_transaction_lifetime << ", " - << "Deferred Transaction Expiration Window: " << c.deferred_trx_expiration_window << ", " - << "Max Transaction Delay: " << c.max_transaction_delay << ", " - << "Max Inline Action Size: " << c.max_inline_action_size << ", " - << "Max Inline Action Depth: " << c.max_inline_action_depth << ", " - << "Max Authority Depth: " << c.max_authority_depth << "\n"; + friend Stream& operator << ( Stream& out, const chain_config_v0& c ) { + return c.log(out) << "\n"; } - friend inline bool operator ==( const chain_config& lhs, const chain_config& rhs ) { + friend inline bool operator ==( const chain_config_v0& lhs, const chain_config_v0& rhs ) { return std::tie( lhs.max_block_net_usage, lhs.target_block_net_usage_pct, lhs.max_transaction_net_usage, @@ -97,13 +108,97 @@ struct chain_config { ); }; - friend inline bool operator !=( const chain_config& lhs, const chain_config& rhs ) { return !(lhs == rhs); } + friend inline bool operator !=( const chain_config_v0& lhs, const chain_config_v0& rhs ) { return !(lhs == rhs); } + +protected: + template + Stream& log(Stream& out) const{ + return out << "Max Block Net Usage: " << max_block_net_usage << ", " + << "Target Block Net Usage Percent: " << ((double)target_block_net_usage_pct / (double)config::percent_1) << "%, " + << "Max Transaction Net Usage: " << max_transaction_net_usage << ", " + << "Base Per-Transaction Net Usage: " << base_per_transaction_net_usage << ", " + << "Net Usage Leeway: " << net_usage_leeway << ", " + << "Context-Free Data Net Usage Discount: " << (double)context_free_discount_net_usage_num * 100.0 / (double)context_free_discount_net_usage_den << "% , " + + << "Max Block CPU Usage: " << max_block_cpu_usage << ", " + << "Target Block CPU Usage Percent: " << ((double)target_block_cpu_usage_pct / (double)config::percent_1) << "%, " + << "Max Transaction CPU Usage: " << max_transaction_cpu_usage << ", " + << "Min Transaction CPU Usage: " << min_transaction_cpu_usage << ", " + + << "Max Transaction Lifetime: " << max_transaction_lifetime << ", " + << "Deferred Transaction Expiration Window: " << deferred_trx_expiration_window << ", " + << "Max Transaction Delay: " << max_transaction_delay << ", " + << "Max Inline Action Size: " << max_inline_action_size << ", " + << "Max Inline Action Depth: " << max_inline_action_depth << ", " + << "Max Authority Depth: " << max_authority_depth; + } +}; + +/** + * @brief v1 Producer-voted blockchain configuration parameters + * + * If Adding new parameters create chain_config_v[n] class instead of adding + * new parameters to v1 or v0. This is needed for snapshots backward compatibility + */ +struct chain_config_v1 : chain_config_v0 { + using Base = chain_config_v0; + + uint32_t max_action_return_value_size = config::default_max_action_return_value_size; ///< size limit for action return value + + //order must match parameters as ids are used in serialization + enum { + max_action_return_value_size_id = Base::PARAMS_COUNT, + PARAMS_COUNT + }; + + inline const Base& base() const { + return static_cast(*this); + } + + void validate() const; + template + friend Stream& operator << ( Stream& out, const chain_config_v1& c ) { + return c.log(out) << "\n"; + } + + friend inline bool operator == ( const chain_config_v1& lhs, const chain_config_v1& rhs ) { + //add v1 parameters comarison here + return std::tie(lhs.max_action_return_value_size) == std::tie(rhs.max_action_return_value_size) + && lhs.base() == rhs.base(); + } + + friend inline bool operator != ( const chain_config_v1& lhs, const chain_config_v1& rhs ) { + return !(lhs == rhs); + } + + inline chain_config_v1& operator= (const Base& b) { + Base::operator= (b); + return *this; + } + +protected: + template + Stream& log(Stream& out) const{ + return base().log(out) << ", Max Action Return Value Size: " << max_action_return_value_size; + } +}; + +class controller; + +struct config_entry_validator{ + const controller& control; + + bool operator()(uint32_t id) const; }; +//after adding 1st value to chain_config_v1 change this using to point to v1 +using chain_config = chain_config_v1; +using config_range = data_range; + } } // namespace eosio::chain -FC_REFLECT(eosio::chain::chain_config, +FC_REFLECT(eosio::chain::chain_config_v0, (max_block_net_usage)(target_block_net_usage_pct) (max_transaction_net_usage)(base_per_transaction_net_usage)(net_usage_leeway) (context_free_discount_net_usage_num)(context_free_discount_net_usage_den) @@ -115,3 +210,272 @@ FC_REFLECT(eosio::chain::chain_config, (max_inline_action_size)(max_inline_action_depth)(max_authority_depth) ) + +FC_REFLECT_DERIVED(eosio::chain::chain_config_v1, (eosio::chain::chain_config_v0), + (max_action_return_value_size) +) + +namespace fc { + +/** + * @brief This is for packing data_entry + * that is used as part of packing data_range + * @param s datastream + * @param entry contains config reference and particular id + * @throws config_parse_error if id is unknown + */ +template +inline DataStream &operator<<(DataStream &s, const eosio::chain::data_entry &entry){ + using namespace eosio::chain; + + //initial requirements were to skip packing field if it is not activated. + //this approach allows to spam this function with big buffer so changing this behavior + EOS_ASSERT(entry.is_allowed(), unsupported_feature, "config id ${id} is no allowed", ("id", entry.id)); + + switch (entry.id){ + case chain_config_v0::max_block_net_usage_id: + fc::raw::pack(s, entry.config.max_block_net_usage); + break; + case chain_config_v0::target_block_net_usage_pct_id: + fc::raw::pack(s, entry.config.target_block_net_usage_pct); + break; + case chain_config_v0::max_transaction_net_usage_id: + fc::raw::pack(s, entry.config.max_transaction_net_usage); + break; + case chain_config_v0::base_per_transaction_net_usage_id: + fc::raw::pack(s, entry.config.base_per_transaction_net_usage); + break; + case chain_config_v0::net_usage_leeway_id: + fc::raw::pack(s, entry.config.net_usage_leeway); + break; + case chain_config_v0::context_free_discount_net_usage_num_id: + fc::raw::pack(s, entry.config.context_free_discount_net_usage_num); + break; + case chain_config_v0::context_free_discount_net_usage_den_id: + fc::raw::pack(s, entry.config.context_free_discount_net_usage_den); + break; + case chain_config_v0::max_block_cpu_usage_id: + fc::raw::pack(s, entry.config.max_block_cpu_usage); + break; + case chain_config_v0::target_block_cpu_usage_pct_id: + fc::raw::pack(s, entry.config.target_block_cpu_usage_pct); + break; + case chain_config_v0::max_transaction_cpu_usage_id: + fc::raw::pack(s, entry.config.max_transaction_cpu_usage); + break; + case chain_config_v0::min_transaction_cpu_usage_id: + fc::raw::pack(s, entry.config.min_transaction_cpu_usage); + break; + case chain_config_v0::max_transaction_lifetime_id: + fc::raw::pack(s, entry.config.max_transaction_lifetime); + break; + case chain_config_v0::deferred_trx_expiration_window_id: + fc::raw::pack(s, entry.config.deferred_trx_expiration_window); + break; + case chain_config_v0::max_transaction_delay_id: + fc::raw::pack(s, entry.config.max_transaction_delay); + break; + case chain_config_v0::max_inline_action_size_id: + fc::raw::pack(s, entry.config.max_inline_action_size); + break; + case chain_config_v0::max_inline_action_depth_id: + fc::raw::pack(s, entry.config.max_inline_action_depth); + break; + case chain_config_v0::max_authority_depth_id: + fc::raw::pack(s, entry.config.max_authority_depth); + break; + default: + FC_THROW_EXCEPTION(config_parse_error, "DataStream& operator<<: no such id: ${id}", ("id", entry.id)); + } + return s; +} + +/** + * @brief This is for packing data_entry + * that is used as part of packing data_range + * @param s datastream + * @param entry contains config reference and particular id + * @throws unsupported_feature if protocol feature for particular id is not activated + */ +template +inline DataStream &operator<<(DataStream &s, const eosio::chain::data_entry &entry){ + using namespace eosio::chain; + + //initial requirements were to skip packing field if it is not activated. + //this approach allows to spam this function with big buffer so changing this behavior + //moreover: + //The contract has no way to know that the value was skipped and is likely to behave incorrectly. + //When the protocol feature is not activated, the old version of nodeos that doesn't know about + //the entry MUST behave the same as the new version of nodeos that does. + //Skipping known but unactivated entries violates this. + EOS_ASSERT(entry.is_allowed(), unsupported_feature, "config id ${id} is no allowed", ("id", entry.id)); + + switch (entry.id){ + case chain_config_v1::max_action_return_value_size_id: + fc::raw::pack(s, entry.config.max_action_return_value_size); + break; + default: + data_entry base_entry(entry); + fc::raw::pack(s, base_entry); + } + + return s; +} + +/** + * @brief This is for unpacking data_entry + * that is used as part of unpacking data_range + * @param s datastream + * @param entry contains config reference and particular id + * @throws unsupported_feature if protocol feature for particular id is not activated + */ +template +inline DataStream &operator>>(DataStream &s, eosio::chain::data_entry &entry){ + using namespace eosio::chain; + + EOS_ASSERT(entry.is_allowed(), eosio::chain::unsupported_feature, "config id ${id} is no allowed", ("id", entry.id)); + + switch (entry.id){ + case chain_config_v0::max_block_net_usage_id: + fc::raw::unpack(s, entry.config.max_block_net_usage); + break; + case chain_config_v0::target_block_net_usage_pct_id: + fc::raw::unpack(s, entry.config.target_block_net_usage_pct); + break; + case chain_config_v0::max_transaction_net_usage_id: + fc::raw::unpack(s, entry.config.max_transaction_net_usage); + break; + case chain_config_v0::base_per_transaction_net_usage_id: + fc::raw::unpack(s, entry.config.base_per_transaction_net_usage); + break; + case chain_config_v0::net_usage_leeway_id: + fc::raw::unpack(s, entry.config.net_usage_leeway); + break; + case chain_config_v0::context_free_discount_net_usage_num_id: + fc::raw::unpack(s, entry.config.context_free_discount_net_usage_num); + break; + case chain_config_v0::context_free_discount_net_usage_den_id: + fc::raw::unpack(s, entry.config.context_free_discount_net_usage_den); + break; + case chain_config_v0::max_block_cpu_usage_id: + fc::raw::unpack(s, entry.config.max_block_cpu_usage); + break; + case chain_config_v0::target_block_cpu_usage_pct_id: + fc::raw::unpack(s, entry.config.target_block_cpu_usage_pct); + break; + case chain_config_v0::max_transaction_cpu_usage_id: + fc::raw::unpack(s, entry.config.max_transaction_cpu_usage); + break; + case chain_config_v0::min_transaction_cpu_usage_id: + fc::raw::unpack(s, entry.config.min_transaction_cpu_usage); + break; + case chain_config_v0::max_transaction_lifetime_id: + fc::raw::unpack(s, entry.config.max_transaction_lifetime); + break; + case chain_config_v0::deferred_trx_expiration_window_id: + fc::raw::unpack(s, entry.config.deferred_trx_expiration_window); + break; + case chain_config_v0::max_transaction_delay_id: + fc::raw::unpack(s, entry.config.max_transaction_delay); + break; + case chain_config_v0::max_inline_action_size_id: + fc::raw::unpack(s, entry.config.max_inline_action_size); + break; + case chain_config_v0::max_inline_action_depth_id: + fc::raw::unpack(s, entry.config.max_inline_action_depth); + break; + case chain_config_v0::max_authority_depth_id: + fc::raw::unpack(s, entry.config.max_authority_depth); + break; + default: + FC_THROW_EXCEPTION(eosio::chain::config_parse_error, "DataStream& operator<<: no such id: ${id}", ("id", entry.id)); + } + + return s; +} + +/** + * @brief This is for unpacking data_entry + * that is used as part of unpacking data_range + * @param s datastream + * @param entry contains config reference and particular id + * @throws unsupported_feature if protocol feature for particular id is not activated + */ +template +inline DataStream &operator>>(DataStream &s, eosio::chain::data_entry &entry){ + using namespace eosio::chain; + + EOS_ASSERT(entry.is_allowed(), unsupported_feature, "config id ${id} is no allowed", ("id", entry.id)); + + switch (entry.id){ + case chain_config_v1::max_action_return_value_size_id: + fc::raw::unpack(s, entry.config.max_action_return_value_size); + break; + default: + eosio::chain::data_entry base_entry(entry); + fc::raw::unpack(s, base_entry); + } + + return s; +} + +/** + * @brief Packs config stream in the following format: + * |uint32_t:sequence_length | uint32_t:parameter_id | :parameter_value | ... + * @param s datastream + * @param selection contains ids range to pack + * @throws config_parse_error on duplicate or unknown id in selection + */ +template +inline DataStream& operator<<( DataStream& s, const eosio::chain::data_range& selection ) { + using namespace eosio::chain; + + fc::unsigned_int size = selection.ids.size(); + fc::raw::pack(s, size); + + //vector here serves as hash map where key is always an index + std::vector visited(T::PARAMS_COUNT, false); + for (auto uid : selection.ids){ + uint32_t id = uid; + EOS_ASSERT(id < visited.size(), config_parse_error, "provided id ${id} should be less than ${size}", ("id", id)("size", visited.size())); + EOS_ASSERT(!visited[id], config_parse_error, "duplicate id provided: ${id}", ("id", id)); + visited[id] = true; + + fc::raw::pack(s, fc::unsigned_int(id)); + fc::raw::pack(s, data_entry(selection.config, id, selection.validator)); + } + + return s; +} + +/** + * @brief Unpacks config stream in the following format: + * |uint32_t:sequence_length | uint32_t:parameter_id | :parameter_value | ... + * @param s datastream + * @param selection contains config reference where values will be unpacked + * @throws config_parse_error on duplicate or unknown id in stream + */ +template +inline DataStream& operator>>( DataStream& s, eosio::chain::data_range& selection ) { + using namespace eosio::chain; + + fc::unsigned_int length; + fc::raw::unpack(s, length); + + //vector here serves as hash map where key is always an index + std::vector visited(T::PARAMS_COUNT, false); + for (uint32_t i = 0; i < length; ++i) { + fc::unsigned_int id; + fc::raw::unpack(s, id); + + EOS_ASSERT(id.value < visited.size(), config_parse_error, "provided id ${id} should be less than ${size}", ("id", id)("size", visited.size())); + EOS_ASSERT(!visited[id], config_parse_error, "duplicate id provided: ${id}", ("id", id)); + visited[id] = true; + + data_entry cfg_entry(selection.config, id, selection.validator); + fc::raw::unpack(s, cfg_entry); + } + return s; +} + +} //namespace fc \ No newline at end of file diff --git a/libraries/chain/include/eosio/chain/chain_config_helper.hpp b/libraries/chain/include/eosio/chain/chain_config_helper.hpp new file mode 100644 index 00000000000..94ab9146419 --- /dev/null +++ b/libraries/chain/include/eosio/chain/chain_config_helper.hpp @@ -0,0 +1,58 @@ +#pragma once + +#include /* vector */ + +namespace eosio { namespace chain { + +/** + * helper class to serialize only selected ids of the class + */ +template +struct data_range { + + T& config; + vector ids; + Validator validator; + + data_range(T& c, Validator val) : config(c), validator(val){} + data_range(T& c, vector&& id_list, const Validator& val) + : data_range(c, val){ + ids = std::move(id_list); + } +}; + +/** + * helper class to serialize specific class entry + */ +template +struct data_entry { +private: + struct _dummy{}; +public: + + T& config; + uint32_t id; + Validator validator; + data_entry(T& c, uint32_t entry_id, Validator validate) + : config(c), + id(entry_id), + validator(validate) {} + template + explicit data_entry(const data_entry& another, + typename std::enable_if_t, _dummy> = _dummy{}) + : data_entry(another.config, another.id, another.validator) + {} + template + explicit data_entry(const data_entry& another, + typename std::enable_if_t, _dummy> = _dummy{}) + : config(std::forward(T{})) { + FC_THROW_EXCEPTION(eosio::chain::config_parse_error, + "this constructor only for compilation of template magic and shouldn't ever be called"); + } + + bool is_allowed() const{ + return validator(id); + } +}; + +}} // namespace eosio::chain \ No newline at end of file diff --git a/libraries/chain/include/eosio/chain/chain_id_type.hpp b/libraries/chain/include/eosio/chain/chain_id_type.hpp index 5a1cc52de86..a6863c74e3c 100644 --- a/libraries/chain/include/eosio/chain/chain_id_type.hpp +++ b/libraries/chain/include/eosio/chain/chain_id_type.hpp @@ -2,8 +2,6 @@ #include -struct hello; - namespace eosio { class net_plugin_impl; @@ -17,6 +15,11 @@ namespace eosio { namespace chain { + namespace legacy { + struct snapshot_global_property_object_v3; + struct snapshot_global_property_object_v4; + } + struct chain_id_type : public fc::sha256 { using fc::sha256::sha256; @@ -34,6 +37,8 @@ namespace chain { void reflector_init()const; + bool empty() const { return *this == chain_id_type{};} + private: chain_id_type() = default; @@ -46,11 +51,14 @@ namespace chain { friend class eosio::net_plugin_impl; friend struct eosio::handshake_message; friend class block_log; - friend struct trim_data; + friend struct block_log_preamble; + friend struct block_log_verifier; friend class controller; friend struct controller_impl; friend class global_property_object; friend struct snapshot_global_property_object; + friend struct legacy::snapshot_global_property_object_v3; + friend struct legacy::snapshot_global_property_object_v4; }; } } // namespace eosio::chain diff --git a/libraries/chain/include/eosio/chain/chain_snapshot.hpp b/libraries/chain/include/eosio/chain/chain_snapshot.hpp index a92e9be8695..d2bd01492f1 100644 --- a/libraries/chain/include/eosio/chain/chain_snapshot.hpp +++ b/libraries/chain/include/eosio/chain/chain_snapshot.hpp @@ -16,10 +16,16 @@ struct chain_snapshot_header { * - WebAuthn keys * - wtmsig block siganatures: the block header state changed to include producer authorities and additional signatures * - removed genesis_state and added chain ID to global_property_object + * 4: Updated for v3.0.0 protocol features: + * - forwards compatible with versions 2 and 3 + * - kv database + * - Configurable wasm limits + * 5: Updated for v3.0.0 eos features: + * - chain_config update */ static constexpr uint32_t minimum_compatible_version = 2; - static constexpr uint32_t current_version = 3; + static constexpr uint32_t current_version = 5; uint32_t version = current_version; diff --git a/libraries/chain/include/eosio/chain/combined_database.hpp b/libraries/chain/include/eosio/chain/combined_database.hpp new file mode 100644 index 00000000000..38647d30b25 --- /dev/null +++ b/libraries/chain/include/eosio/chain/combined_database.hpp @@ -0,0 +1,153 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +// It's a fatal condition if chainbase and chain_kv get out of sync with each +// other due to exceptions. +#define CATCH_AND_EXIT_DB_FAILURE() \ + catch (...) { \ + elog("Error while using database"); \ + std::abort(); \ + } + +namespace eosio { namespace chain { + using rocks_db_type = eosio::session::session; + using session_type = eosio::session::session; + using kv_undo_stack_ptr = std::unique_ptr>; + + using controller_index_set = + index_set; + + using contract_database_index_set = index_set; + + class apply_context; + + namespace backing_store { + struct db_context; + } + using db_context = backing_store::db_context; + + class combined_session { + public: + combined_session() = default; + + combined_session(chainbase::database& cb_database, eosio::session::undo_stack* undo_stack); + + combined_session(combined_session&& src) noexcept; + + ~combined_session() { undo(); } + + combined_session& operator=(const combined_session& src) = delete; + + void push(); + + void squash(); + + void undo(); + + private: + std::unique_ptr cb_session = {}; + eosio::session::undo_stack* kv_undo_stack = nullptr; + }; + + class combined_database { + public: + explicit combined_database(chainbase::database& chain_db, + uint32_t snapshot_batch_threashold); + + combined_database(chainbase::database& chain_db, + const controller::config& cfg); + + combined_database(const combined_database& copy) = delete; + combined_database& operator=(const combined_database& copy) = delete; + + // Save the backing_store setting to the chainbase in order to detect + // when this setting is switched between chainbase and rocksdb. + // If existing state is not clean, switching is not allowed. + void check_backing_store_setting(bool clean_startup); + + static combined_session make_no_op_session() { return combined_session(); } + + combined_session make_session() { + return combined_session(db, kv_undo_stack.get()); + } + + void set_revision(uint64_t revision); + + int64_t revision(); + + void undo(); + + void commit(int64_t revision); + + void flush(); + + static void destroy(const fc::path& p); + + std::unique_ptr create_kv_context(name receiver, kv_resource_manager resource_manager, + const kv_database_config& limits)const; + + std::unique_ptr create_db_context(apply_context& context, name receiver); + + void add_to_snapshot(const eosio::chain::snapshot_writer_ptr& snapshot, const eosio::chain::block_state& head, + const eosio::chain::authorization_manager& authorization, + const eosio::chain::resource_limits::resource_limits_manager& resource_limits) const; + + void read_from_snapshot(const snapshot_reader_ptr& snapshot, uint32_t blog_start, uint32_t blog_end, + eosio::chain::authorization_manager& authorization, + eosio::chain::resource_limits::resource_limits_manager& resource_limits, + eosio::chain::fork_database& fork_db, eosio::chain::block_state_ptr& head, + uint32_t& snapshot_head_block, const eosio::chain::chain_id_type& chain_id); + + auto &get_db(void) const { return db; } + auto &get_kv_undo_stack(void) const { return kv_undo_stack; } + backing_store_type get_backing_store() const { return backing_store; } + + private: + void add_contract_tables_to_snapshot(const snapshot_writer_ptr& snapshot) const; + void read_contract_tables_from_snapshot(const snapshot_reader_ptr& snapshot); + + backing_store_type backing_store; + chainbase::database& db; + std::unique_ptr kv_database; + kv_undo_stack_ptr kv_undo_stack; + const uint64_t kv_snapshot_batch_threashold; + }; + + std::optional extract_legacy_genesis_state(snapshot_reader& snapshot, uint32_t version); + + std::vector make_rocksdb_contract_kv_prefix(); + char make_rocksdb_contract_db_prefix(); + +}} // namespace eosio::chain diff --git a/libraries/chain/include/eosio/chain/config.hpp b/libraries/chain/include/eosio/chain/config.hpp index 6950b534468..e88d3abb04f 100644 --- a/libraries/chain/include/eosio/chain/config.hpp +++ b/libraries/chain/include/eosio/chain/config.hpp @@ -1,6 +1,7 @@ #pragma once #include #include +#include #pragma GCC diagnostic ignored "-Wunused-variable" @@ -9,7 +10,10 @@ namespace eosio { namespace chain { namespace config { typedef __uint128_t uint128_t; const static auto default_blocks_dir_name = "blocks"; -const static auto reversible_blocks_dir_name = "reversible"; +const static auto default_blocks_archive_dir_name = "archive"; +const static auto default_blocks_log_stride = UINT32_MAX; +const static auto default_max_retained_block_files = 10; +const static auto reversible_blocks_dir_name = "reversible"; const static auto default_reversible_cache_size = 340*1024*1024ll;/// 1MB * 340 blocks based on 21 producer BFT delay const static auto default_reversible_guard_size = 2*1024*1024ll;/// 1MB * 340 blocks based on 21 producer BFT delay @@ -19,21 +23,21 @@ const static auto default_state_size = 1*1024*1024*1024ll; const static auto default_state_guard_size = 128*1024*1024ll; -const static name system_account_name { N(eosio) }; -const static name null_account_name { N(eosio.null) }; -const static name producers_account_name { N(eosio.prods) }; +const static name system_account_name { "eosio"_n }; +const static name null_account_name { "eosio.null"_n }; +const static name producers_account_name { "eosio.prods"_n }; // Active permission of producers account requires greater than 2/3 of the producers to authorize -const static name majority_producers_permission_name { N(prod.major) }; // greater than 1/2 of producers needed to authorize -const static name minority_producers_permission_name { N(prod.minor) }; // greater than 1/3 of producers needed to authorize0 +const static name majority_producers_permission_name { "prod.major"_n }; // greater than 1/2 of producers needed to authorize +const static name minority_producers_permission_name { "prod.minor"_n }; // greater than 1/3 of producers needed to authorize0 -const static name eosio_auth_scope { N(eosio.auth) }; -const static name eosio_all_scope { N(eosio.all) }; +const static name eosio_auth_scope { "eosio.auth"_n }; +const static name eosio_all_scope { "eosio.all"_n }; -const static name active_name { N(active) }; -const static name owner_name { N(owner) }; -const static name eosio_any_name { N(eosio.any) }; -const static name eosio_code_name { N(eosio.code) }; +const static name active_name { "active"_n }; +const static name owner_name { "owner"_n }; +const static name eosio_any_name { "eosio.any"_n }; +const static name eosio_code_name { "eosio.code"_n }; const static int block_interval_ms = 500; const static int block_interval_us = block_interval_ms*1000; @@ -83,6 +87,30 @@ const static uint32_t default_block_cpu_effort_pct = 80 * perc const static uint16_t default_controller_thread_pool_size = 2; const static uint32_t default_max_variable_signature_length = 16384u; const static uint32_t default_max_nonprivileged_inline_action_size = 4 * 1024; // 4 KB +const static uint32_t default_max_action_return_value_size = 256; +const static uint16_t default_persistent_storage_num_threads = 1; +const static int default_persistent_storage_max_num_files = -1; +const static uint64_t default_persistent_storage_write_buffer_size = 128 * 1024 * 1024; +const static uint64_t default_persistent_storage_bytes_per_sync = 1 * 1024 * 1024; +const static uint32_t default_persistent_storage_mbytes_batch = 50; + +static_assert(MAX_SIZE_OF_BYTE_ARRAYS == 20*1024*1024, "Changing MAX_SIZE_OF_BYTE_ARRAYS breaks consensus. Make sure this is expected"); + +const static uint32_t default_max_kv_key_size = 1024; +const static uint32_t default_max_kv_value_size = 1024*1024; // Large enough to hold most contracts +const static uint32_t default_max_kv_iterators = 1024; + +const static uint32_t default_max_wasm_mutable_global_bytes = 1024; +const static uint32_t default_max_wasm_table_elements = 1024; +const static uint32_t default_max_wasm_section_elements = 8192; +const static uint32_t default_max_wasm_linear_memory_init = 64*1024; +const static uint32_t default_max_wasm_func_local_bytes = 8192; +const static uint32_t default_max_wasm_nested_structures = 1024; +const static uint32_t default_max_wasm_symbol_bytes = 8192; +const static uint32_t default_max_wasm_module_bytes = 20*1024*1024; +const static uint32_t default_max_wasm_code_bytes = 20*1024*1024; +const static uint32_t default_max_wasm_pages = 528; +const static uint32_t default_max_wasm_call_depth = 251; const static uint32_t min_net_usage_delta_between_base_and_max_for_trx = 10*1024; // Should be large enough to allow recovery from badly set blockchain parameters without a hard fork @@ -97,7 +125,7 @@ const static uint32_t setcode_ram_bytes_multiplier = 10; ///< multip const static uint32_t hashing_checktime_block_size = 10*1024; /// call checktime from hashing intrinsic once per this number of bytes -const static eosio::chain::wasm_interface::vm_type default_wasm_runtime = eosio::chain::wasm_interface::vm_type::wabt; +const static eosio::chain::wasm_interface::vm_type default_wasm_runtime = eosio::chain::wasm_interface::vm_type::eos_vm_jit; const static uint32_t default_abi_serializer_max_time_us = 15*1000; ///< default deadline for abi serialization methods /** diff --git a/libraries/chain/include/eosio/chain/contract_table_objects.hpp b/libraries/chain/include/eosio/chain/contract_table_objects.hpp index baf549a2b13..1ffc8cc0de2 100644 --- a/libraries/chain/include/eosio/chain/contract_table_objects.hpp +++ b/libraries/chain/include/eosio/chain/contract_table_objects.hpp @@ -294,6 +294,7 @@ CHAINBASE_SET_INDEX_TYPE(eosio::chain::index_double_object, eosio::chain::index_ CHAINBASE_SET_INDEX_TYPE(eosio::chain::index_long_double_object, eosio::chain::index_long_double_index) FC_REFLECT(eosio::chain::table_id_object, (code)(scope)(table)(payer)(count) ) +// @ignore t_id FC_REFLECT(eosio::chain::key_value_object, (primary_key)(payer)(value) ) #define REFLECT_SECONDARY(type)\ diff --git a/libraries/chain/include/eosio/chain/contract_types.hpp b/libraries/chain/include/eosio/chain/contract_types.hpp index 3870f9b8440..54c0f8212ba 100644 --- a/libraries/chain/include/eosio/chain/contract_types.hpp +++ b/libraries/chain/include/eosio/chain/contract_types.hpp @@ -18,7 +18,7 @@ struct newaccount { } static action_name get_name() { - return N(newaccount); + return "newaccount"_n; } }; @@ -33,7 +33,7 @@ struct setcode { } static action_name get_name() { - return N(setcode); + return "setcode"_n; } }; @@ -46,7 +46,7 @@ struct setabi { } static action_name get_name() { - return N(setabi); + return "setabi"_n; } }; @@ -62,7 +62,7 @@ struct updateauth { } static action_name get_name() { - return N(updateauth); + return "updateauth"_n; } }; @@ -80,7 +80,7 @@ struct deleteauth { } static action_name get_name() { - return N(deleteauth); + return "deleteauth"_n; } }; @@ -100,7 +100,7 @@ struct linkauth { } static action_name get_name() { - return N(linkauth); + return "linkauth"_n; } }; @@ -119,7 +119,7 @@ struct unlinkauth { } static action_name get_name() { - return N(unlinkauth); + return "unlinkauth"_n; } }; @@ -132,7 +132,7 @@ struct canceldelay { } static action_name get_name() { - return N(canceldelay); + return "canceldelay"_n; } }; @@ -148,7 +148,7 @@ struct onerror { } static action_name get_name() { - return N(onerror); + return "onerror"_n; } }; diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index 2d08e4239a4..ea56eb0278d 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -7,17 +7,24 @@ #include #include -#include #include #include +#include +#include namespace chainbase { class database; } +namespace b1::chain_kv { + struct database; + class undo_stack; +} namespace boost { namespace asio { class thread_pool; }} +namespace eosio { namespace vm { class wasm_allocator; }} + namespace eosio { namespace chain { class authorization_manager; @@ -26,6 +33,8 @@ namespace eosio { namespace chain { class resource_limits_manager; }; + class combined_database; + struct controller_impl; using chainbase::database; using chainbase::pinnable_mapped_file; @@ -65,22 +74,33 @@ namespace eosio { namespace chain { flat_set contract_blacklist; flat_set< pair > action_blacklist; flat_set key_blacklist; - path blocks_dir = chain::config::default_blocks_dir_name; - path state_dir = chain::config::default_state_dir_name; - uint64_t state_size = chain::config::default_state_size; - uint64_t state_guard_size = chain::config::default_state_guard_size; - uint64_t reversible_cache_size = chain::config::default_reversible_cache_size; - uint64_t reversible_guard_size = chain::config::default_reversible_guard_size; - uint32_t sig_cpu_bill_pct = chain::config::default_sig_cpu_bill_pct; - uint16_t thread_pool_size = chain::config::default_controller_thread_pool_size; + block_log_config blog; + path state_dir = chain::config::default_state_dir_name; + uint64_t state_size = chain::config::default_state_size; + uint64_t state_guard_size = chain::config::default_state_guard_size; + uint64_t reversible_cache_size = chain::config::default_reversible_cache_size; + uint64_t reversible_guard_size = chain::config::default_reversible_guard_size; + uint32_t sig_cpu_bill_pct = chain::config::default_sig_cpu_bill_pct; + uint16_t thread_pool_size = chain::config::default_controller_thread_pool_size; + uint16_t max_retained_block_files = chain::config::default_max_retained_block_files; + uint64_t blocks_log_stride = chain::config::default_blocks_log_stride; + backing_store_type backing_store = backing_store_type::CHAINBASE; + uint16_t persistent_storage_num_threads = 0; // Will be set to number of cores dynamically or by user configuration; + int persistent_storage_max_num_files = chain::config::default_persistent_storage_max_num_files; + uint64_t persistent_storage_write_buffer_size = chain::config::default_persistent_storage_write_buffer_size; + uint64_t persistent_storage_bytes_per_sync = chain::config::default_persistent_storage_bytes_per_sync; + uint32_t persistent_storage_mbytes_batch = chain::config::default_persistent_storage_mbytes_batch; + fc::microseconds abi_serializer_max_time_us = fc::microseconds(chain::config::default_abi_serializer_max_time_us); uint32_t max_nonprivileged_inline_action_size = chain::config::default_max_nonprivileged_inline_action_size; - bool read_only = false; - bool force_all_checks = false; - bool disable_replay_opts = false; - bool contracts_console = false; + bool read_only = false; + bool force_all_checks = false; + bool disable_replay_opts = false; + bool contracts_console = false; bool allow_ram_billing_in_notify = false; + uint32_t maximum_variable_signature_length = chain::config::default_max_variable_signature_length; - bool disable_all_subjective_mitigations = false; //< for testing purposes only + bool disable_all_subjective_mitigations = false; //< for developer & testing purposes, can be configured using `disable-all-subjective-mitigations` when `EOSIO_DEVELOPER` build option is provided + uint32_t terminate_at_block = 0; //< primarily for testing purposes wasm_interface::vm_type wasm_runtime = chain::config::default_wasm_runtime; eosvmoc::config eosvmoc_config; @@ -90,7 +110,6 @@ namespace eosio { namespace chain { validation_mode block_validation_mode = validation_mode::FULL; pinnable_mapped_file::map_mode db_map_mode = pinnable_mapped_file::map_mode::mapped; - vector db_hugepage_paths; flat_set resource_greylist; flat_set trusted_producers; @@ -104,16 +123,17 @@ namespace eosio { namespace chain { incomplete = 3, ///< this is an incomplete block (either being produced by a producer or speculatively produced by a node) }; + controller( const config& cfg, const chain_id_type& chain_id ); controller( const config& cfg, protocol_feature_set&& pfs, const chain_id_type& chain_id ); ~controller(); void add_indices(); - void startup( std::function shutdown, const snapshot_reader_ptr& snapshot); - void startup( std::function shutdown, const genesis_state& genesis); - void startup( std::function shutdown); + void startup( std::function shutdown, std::function check_shutdown, const snapshot_reader_ptr& snapshot); + void startup( std::function shutdown, std::function check_shutdown, const genesis_state& genesis); + void startup( std::function shutdown, std::function check_shutdown); - void preactivate_feature( const digest_type& feature_digest ); + void preactivate_feature( uint32_t action_id, const digest_type& feature_digest ); vector get_preactivated_protocol_features()const; @@ -138,7 +158,7 @@ namespace eosio { namespace chain { /** * @return transactions applied in aborted block */ - vector abort_block(); + deque abort_block(); /** * @@ -158,20 +178,21 @@ namespace eosio { namespace chain { void sign_block( const signer_callback_type& signer_callback ); void commit_block(); - std::future create_block_state_future( const signed_block_ptr& b ); + std::future create_block_state_future( const block_id_type& id, const signed_block_ptr& b ); /** * @param block_state_future provide from call to create_block_state_future * @param cb calls cb with forked applied transactions for each forked block * @param trx_lookup user provided lookup function for externally cached transaction_metadata */ - void push_block( std::future& block_state_future, + block_state_ptr push_block( std::future& block_state_future, const forked_branch_callback& cb, const trx_meta_cache_lookup& trx_lookup ); boost::asio::io_context& get_thread_pool(); const chainbase::database& db()const; + const chainbase::database& reversible_db() const; const fork_database& fork_db()const; @@ -184,6 +205,7 @@ namespace eosio { namespace chain { authorization_manager& get_mutable_authorization_manager(); const protocol_feature_manager& get_protocol_feature_manager()const; uint32_t get_max_nonprivileged_inline_action_size()const; + const config& get_config()const; const flat_set& get_actor_whitelist() const; const flat_set& get_actor_blacklist() const; @@ -219,17 +241,18 @@ namespace eosio { namespace chain { time_point pending_block_time()const; account_name pending_block_producer()const; const block_signing_authority& pending_block_signing_authority()const; - optional pending_producer_block_id()const; + std::optional pending_producer_block_id()const; - const vector& get_pending_trx_receipts()const; + const deque& get_pending_trx_receipts()const; - const producer_authority_schedule& active_producers()const; - const producer_authority_schedule& pending_producers()const; - optional proposed_producers()const; + const producer_authority_schedule& active_producers()const; + const producer_authority_schedule& pending_producers()const; + std::optional proposed_producers()const; uint32_t last_irreversible_block_num() const; block_id_type last_irreversible_block_id() const; time_point last_irreversible_block_time() const; + const signed_block_ptr last_irreversible_block() const; signed_block_ptr fetch_block_by_number( uint32_t block_num )const; signed_block_ptr fetch_block_by_id( block_id_type id )const; @@ -274,11 +297,11 @@ namespace eosio { namespace chain { int64_t set_proposed_producers( vector producers ); - bool light_validation_allowed(bool replay_opts_disabled_by_policy) const; + bool light_validation_allowed() const; bool skip_auth_check()const; - bool skip_db_sessions( )const; - bool skip_db_sessions( block_status bs )const; bool skip_trx_checks()const; + bool skip_db_sessions()const; + bool skip_db_sessions( block_status bs )const; bool is_trusted_producer( const account_name& producer) const; bool contracts_console()const; @@ -287,20 +310,27 @@ namespace eosio { namespace chain { db_read_mode get_read_mode()const; validation_mode get_validation_mode()const; + const flat_set& get_trusted_producers()const; + uint32_t get_terminate_at_block()const; void set_subjective_cpu_leeway(fc::microseconds leeway); - fc::optional get_subjective_cpu_leeway() const; + std::optional get_subjective_cpu_leeway() const; void set_greylist_limit( uint32_t limit ); uint32_t get_greylist_limit()const; - void add_to_ram_correction( account_name account, uint64_t ram_bytes ); + fc::microseconds get_abi_serializer_max_time() const; + + void add_to_ram_correction( account_name account, uint64_t ram_bytes, uint32_t action_id, const char* event_id ); bool all_subjective_mitigations_disabled()const; + fc::logger* get_deep_mind_logger() const; + void enable_deep_mind( fc::logger* logger ); + #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED) vm::wasm_allocator& get_wasm_allocator(); #endif - static fc::optional convert_exception_to_error_code( const fc::exception& e ); + static std::optional convert_exception_to_error_code( const fc::exception& e ); signal block_start; // block_num signal pre_accepted_block; @@ -308,7 +338,7 @@ namespace eosio { namespace chain { signal accepted_block; signal irreversible_block; signal accepted_transaction; - signal)> applied_transaction; + signal)> applied_transaction; signal bad_alloc; /* @@ -325,7 +355,7 @@ namespace eosio { namespace chain { wasm_interface& get_wasm_interface(); - optional get_abi_serializer( account_name n, const abi_serializer::yield_function_t& yield )const { + std::optional get_abi_serializer( account_name n, const abi_serializer::yield_function_t& yield )const { if( n.good() ) { try { const auto& a = get_account( n ); @@ -334,7 +364,7 @@ namespace eosio { namespace chain { return abi_serializer( abi, yield ); } FC_CAPTURE_AND_LOG((n)) } - return optional(); + return std::optional(); } template @@ -345,9 +375,24 @@ namespace eosio { namespace chain { return pretty_output; } + template + fc::variant maybe_to_variant_with_abi( const T& obj, const abi_serializer::yield_function_t& yield ) { + try { + return to_variant_with_abi(obj, yield); + } FC_LOG_AND_DROP() + + // If we are unable to transform to an ABI aware variant, let's just return the original `obj` as-is + return fc::variant(obj); + } + static chain_id_type extract_chain_id(snapshot_reader& snapshot); - static fc::optional extract_chain_id_from_db( const path& state_dir ); + static std::optional extract_chain_id_from_db( const path& state_dir ); + + void replace_producer_keys( const public_key_type& key ); + void replace_account_keys( name account, name permission, const public_key_type& key ); + + eosio::chain::combined_database& kv_db()const; private: friend class apply_context; diff --git a/libraries/chain/include/eosio/chain/database_utils.hpp b/libraries/chain/include/eosio/chain/database_utils.hpp index 84c90842795..dbca3aa454c 100644 --- a/libraries/chain/include/eosio/chain/database_utils.hpp +++ b/libraries/chain/include/eosio/chain/database_utils.hpp @@ -24,6 +24,16 @@ namespace eosio { namespace chain { } } + template + static void walk( const chainbase::database& db, F function ) { + auto const& index = db.get_index(); + const auto& first = index.begin(); + const auto& last = index.end(); + for (auto itr = first; itr != last; ++itr) { + function(*itr); + } + } + template static void walk_range( const chainbase::database& db, const Key& begin_key, const Key& end_key, F function ) { const auto& idx = db.get_index(); @@ -106,35 +116,61 @@ namespace fc { from_variant(v, oid._id); } + inline + void float64_to_double (const float64_t& f, double& d) { + memcpy(&d, &f, sizeof(d)); + } + + inline + void double_to_float64 (const double& d, float64_t& f) { + memcpy(&f, &d, sizeof(f)); + } + + inline + void float128_to_uint128 (const float128_t& f, eosio::chain::uint128_t& u) { + memcpy(&u, &f, sizeof(u)); + } + + inline + void uint128_to_float128 (const eosio::chain::uint128_t& u, float128_t& f) { + memcpy(&f, &u, sizeof(f)); + } + inline void to_variant( const float64_t& f, variant& v ) { - v = variant(*reinterpret_cast(&f)); + double double_f; + float64_to_double(f, double_f); + v = variant(double_f); } inline void from_variant( const variant& v, float64_t& f ) { - from_variant(v, *reinterpret_cast(&f)); + double double_f; + from_variant(v, double_f); + double_to_float64(double_f, f); } inline void to_variant( const float128_t& f, variant& v ) { // Assumes platform is little endian and hex representation of 128-bit integer is in little endian order. - const eosio::chain::uint128_t as_bytes = *reinterpret_cast(&f); + char as_bytes[sizeof(eosio::chain::uint128_t)]; + memcpy(as_bytes, &f, sizeof(as_bytes)); std::string s = "0x"; - s.append( to_hex( reinterpret_cast(&as_bytes), sizeof(as_bytes) ) ); + s.append( to_hex( as_bytes, sizeof(as_bytes) ) ); v = s; } inline void from_variant( const variant& v, float128_t& f ) { // Temporarily hold the binary in uint128_t before casting it to float128_t - eosio::chain::uint128_t temp = 0; + char temp[sizeof(eosio::chain::uint128_t)]; + memset(temp, 0, sizeof(temp)); auto s = v.as_string(); FC_ASSERT( s.size() == 2 + 2 * sizeof(temp) && s.find("0x") == 0, "Failure in converting hex data into a float128_t"); - auto sz = from_hex( s.substr(2), reinterpret_cast(&temp), sizeof(temp) ); + auto sz = from_hex( s.substr(2), temp, sizeof(temp) ); // Assumes platform is little endian and hex representation of 128-bit integer is in little endian order. FC_ASSERT( sz == sizeof(temp), "Failure in converting hex data into a float128_t" ); - f = *reinterpret_cast(&temp); + memcpy(&f, temp, sizeof(f)); } inline @@ -202,24 +238,32 @@ namespace chainbase { // overloads for softfloat packing template DataStream& operator << ( DataStream& ds, const float64_t& v ) { - fc::raw::pack(ds, *reinterpret_cast(&v)); + double double_v; + fc::float64_to_double(v, double_v); + fc::raw::pack(ds, double_v); return ds; } template DataStream& operator >> ( DataStream& ds, float64_t& v ) { - fc::raw::unpack(ds, *reinterpret_cast(&v)); + double double_v; + fc::raw::unpack(ds, double_v); + fc::double_to_float64(double_v, v); return ds; } template DataStream& operator << ( DataStream& ds, const float128_t& v ) { - fc::raw::pack(ds, *reinterpret_cast(&v)); + eosio::chain::uint128_t uint128_v; + fc::float128_to_uint128(v, uint128_v); + fc::raw::pack(ds, uint128_v); return ds; } template DataStream& operator >> ( DataStream& ds, float128_t& v ) { - fc::raw::unpack(ds, *reinterpret_cast(&v)); + eosio::chain::uint128_t uint128_v; + fc::raw::unpack(ds, uint128_v); + fc::uint128_to_float128(uint128_v, v); return ds; } diff --git a/libraries/chain/include/eosio/chain/exceptions.hpp b/libraries/chain/include/eosio/chain/exceptions.hpp index 528fa437c18..c4507d42fe4 100644 --- a/libraries/chain/include/eosio/chain/exceptions.hpp +++ b/libraries/chain/include/eosio/chain/exceptions.hpp @@ -125,7 +125,7 @@ { if( code() == CODE ) throw *this;\ else fc::exception::dynamic_rethrow_exception(); \ } \ - fc::optional error_code; \ + std::optional error_code; \ }; namespace eosio { namespace chain { @@ -159,8 +159,8 @@ namespace eosio { namespace chain { * |- reversible_blocks_exception * |- block_log_exception * |- resource_limit_exception - * |- mongo_db_exception * |- contract_api_exception + * |- state_history_exception */ FC_DECLARE_DERIVED_EXCEPTION( chain_type_exception, chain_exception, @@ -273,10 +273,14 @@ namespace eosio { namespace chain { 3040015, "Invalid transaction extension" ) FC_DECLARE_DERIVED_EXCEPTION( ill_formed_deferred_transaction_generation_context, transaction_exception, 3040016, "Transaction includes an ill-formed deferred transaction generation context extension" ) - FC_DECLARE_DERIVED_EXCEPTION( disallowed_transaction_extensions_bad_block_exception, transaction_exception, - 3040017, "Transaction includes disallowed extensions (invalid block)" ) FC_DECLARE_DERIVED_EXCEPTION( tx_resource_exhaustion, transaction_exception, 3040018, "Transaction exceeded transient resource limit" ) + FC_DECLARE_DERIVED_EXCEPTION( tx_prune_exception, transaction_exception, + 3040019, "Prunable data not found" ) + FC_DECLARE_DERIVED_EXCEPTION( tx_no_signature, transaction_exception, + 3040020, "Transaction signatures pruned" ) + FC_DECLARE_DERIVED_EXCEPTION( tx_no_context_free_data, transaction_exception, + 3040021, "Transaction context free data pruned" ) FC_DECLARE_DERIVED_EXCEPTION( action_validate_exception, chain_exception, @@ -306,7 +310,11 @@ namespace eosio { namespace chain { 3050011, "eosio_assert_code assertion failure uses restricted error code value" ) FC_DECLARE_DERIVED_EXCEPTION( inline_action_too_big_nonprivileged, action_validate_exception, 3050012, "Inline action exceeds maximum size limit for a non-privileged account" ) - + FC_DECLARE_DERIVED_EXCEPTION( unauthorized_disk_usage_increase, action_validate_exception, + 3050013, "Action attempts to increase disk usage of account without authorization" ) + FC_DECLARE_DERIVED_EXCEPTION( action_return_value_exception, action_validate_exception, + 3050014, "action return value size too big" ) + FC_DECLARE_DERIVED_EXCEPTION( database_exception, chain_exception, 3060000, "Database exception" ) @@ -320,6 +328,16 @@ namespace eosio { namespace chain { 3060004, "Contract Query Exception" ) FC_DECLARE_DERIVED_EXCEPTION( bad_database_version_exception, database_exception, 3060005, "Database is an unknown or unsupported version" ) + FC_DECLARE_DERIVED_EXCEPTION( database_revision_mismatch_exception, database_exception, + 3060006, "Chainbase and chain-kv databases are at different revisions" ) + FC_DECLARE_DERIVED_EXCEPTION( database_move_kv_disk_exception, database_exception, + 3060007, "Cannot change backing store when existing state has already stored data in a different backing store; use resync, replay, or snapshot to move these to the new backing store" ) + FC_DECLARE_DERIVED_EXCEPTION( kv_rocksdb_bad_value_size_exception, database_exception, + 3060008, "The size of value returned from RocksDB is less than payer's size" ) + FC_DECLARE_DERIVED_EXCEPTION( bad_composite_key_exception, database_exception, + 3060009, "Retrieved composite key from key/value store that was formatted incorrectly" ) + FC_DECLARE_DERIVED_EXCEPTION( db_rocksdb_invalid_operation_exception, database_exception, + 3060010, "Requested operation not valid for database state." ) FC_DECLARE_DERIVED_EXCEPTION( guard_exception, database_exception, 3060100, "Guard Exception" ) @@ -362,6 +380,8 @@ namespace eosio { namespace chain { 3080007, "Transaction exceeded the current greylisted account network usage limit" ) FC_DECLARE_DERIVED_EXCEPTION( greylist_cpu_usage_exceeded, resource_exhausted_exception, 3080008, "Transaction exceeded the current greylisted account CPU usage limit" ) + FC_DECLARE_DERIVED_EXCEPTION( disk_usage_exceeded, resource_exhausted_exception, + 3080009, "Account using more than allotted DISK usage" ) FC_DECLARE_DERIVED_EXCEPTION( leeway_deadline_exception, deadline_exception, 3081001, "Transaction reached the deadline set due to leeway on account CPU limits" ) @@ -400,8 +420,6 @@ namespace eosio { namespace chain { 3100004, "Corrupted reversible block database was fixed" ) FC_DECLARE_DERIVED_EXCEPTION( extract_genesis_state_exception, misc_exception, 3100005, "Extracted genesis state from blocks.log" ) - FC_DECLARE_DERIVED_EXCEPTION( subjective_block_production_exception, misc_exception, - 3100006, "Subjective exception thrown during block production" ) FC_DECLARE_DERIVED_EXCEPTION( multiple_voter_info, misc_exception, 3100007, "Multiple voter info detected" ) FC_DECLARE_DERIVED_EXCEPTION( unsupported_feature, misc_exception, @@ -480,7 +498,8 @@ namespace eosio { namespace chain { 3140000, "Exceptions that are allowed to bubble out of emit calls in controller" ) FC_DECLARE_DERIVED_EXCEPTION( checkpoint_exception, controller_emit_signal_exception, 3140001, "Block does not match checkpoint" ) - + FC_DECLARE_DERIVED_EXCEPTION( state_history_write_exception, controller_emit_signal_exception, + 3140002, "State history write error" ) FC_DECLARE_DERIVED_EXCEPTION( abi_exception, chain_exception, 3015000, "ABI exception" ) @@ -516,6 +535,10 @@ namespace eosio { namespace chain { 3015015, "Duplicate variant definition in the ABI" ) FC_DECLARE_DERIVED_EXCEPTION( unsupported_abi_version_exception, abi_exception, 3015016, "ABI has an unsupported version" ) + FC_DECLARE_DERIVED_EXCEPTION( duplicate_abi_action_results_def_exception, abi_exception, + 3015017, "Duplicate action results definition in the ABI" ) + FC_DECLARE_DERIVED_EXCEPTION(duplicate_abi_kv_table_def_exception, abi_exception, + 3015018, "Duplicate kv_table definition in the ABI") FC_DECLARE_DERIVED_EXCEPTION( contract_exception, chain_exception, 3160000, "Contract exception" ) @@ -539,6 +562,18 @@ namespace eosio { namespace chain { 3160009, "No wasm file found" ) FC_DECLARE_DERIVED_EXCEPTION( abi_file_not_found, contract_exception, 3160010, "No abi file found" ) + FC_DECLARE_DERIVED_EXCEPTION( kv_bad_db_id, contract_exception, + 3160011, "Bad key-value database ID" ) + FC_DECLARE_DERIVED_EXCEPTION( kv_bad_iter, contract_exception, + 3160012, "Bad key-value iterator" ) + FC_DECLARE_DERIVED_EXCEPTION( kv_limit_exceeded, contract_exception, + 3160013, "The key or value is too large" ) + FC_DECLARE_DERIVED_EXCEPTION( kv_unknown_parameters_version, contract_exception, + 3160014, "Unknown kv_parameters version" ) + FC_DECLARE_DERIVED_EXCEPTION( wasm_config_unknown_version, contract_exception, + 3160015, "Unknown wasm_config version" ) + FC_DECLARE_DERIVED_EXCEPTION( config_parse_error, contract_exception, + 3160015, "Parsing config error" ) FC_DECLARE_DERIVED_EXCEPTION( producer_exception, chain_exception, 3170000, "Producer exception" ) @@ -564,6 +599,8 @@ namespace eosio { namespace chain { 3170011, "The signer returned no valid block signatures" ) FC_DECLARE_DERIVED_EXCEPTION( unsupported_multiple_block_signatures, producer_exception, 3170012, "The signer returned multiple signatures but that is not supported" ) + FC_DECLARE_DERIVED_EXCEPTION( block_validation_error, producer_exception, + 3170013, "Block Validation Exception" ) FC_DECLARE_DERIVED_EXCEPTION( reversible_blocks_exception, chain_exception, 3180000, "Reversible Blocks exception" ) @@ -605,13 +642,6 @@ namespace eosio { namespace chain { FC_DECLARE_DERIVED_EXCEPTION( resource_limit_exception, chain_exception, 3210000, "Resource limit exception" ) - FC_DECLARE_DERIVED_EXCEPTION( mongo_db_exception, chain_exception, - 3220000, "Mongo DB exception" ) - FC_DECLARE_DERIVED_EXCEPTION( mongo_db_insert_fail, mongo_db_exception, - 3220001, "Fail to insert new data to Mongo DB" ) - FC_DECLARE_DERIVED_EXCEPTION( mongo_db_update_fail, mongo_db_exception, - 3220002, "Fail to update existing data in Mongo DB" ) - FC_DECLARE_DERIVED_EXCEPTION( contract_api_exception, chain_exception, 3230000, "Contract API exception" ) FC_DECLARE_DERIVED_EXCEPTION( crypto_api_exception, contract_api_exception, @@ -625,13 +655,31 @@ namespace eosio { namespace chain { 3240000, "Snapshot exception" ) FC_DECLARE_DERIVED_EXCEPTION( snapshot_validation_exception, snapshot_exception, 3240001, "Snapshot Validation Exception" ) + FC_DECLARE_DERIVED_EXCEPTION( snapshot_decompress_exception, snapshot_exception, + 3240002, "Snapshot decompress error" ) FC_DECLARE_DERIVED_EXCEPTION( protocol_feature_exception, chain_exception, 3250000, "Protocol feature exception" ) FC_DECLARE_DERIVED_EXCEPTION( protocol_feature_validation_exception, protocol_feature_exception, 3250001, "Protocol feature validation exception" ) - FC_DECLARE_DERIVED_EXCEPTION( protocol_feature_bad_block_exception, protocol_feature_exception, - 3250002, "Protocol feature exception (invalid block)" ) FC_DECLARE_DERIVED_EXCEPTION( protocol_feature_iterator_exception, protocol_feature_exception, 3250003, "Protocol feature iterator exception" ) + + // Any derived types of subjective_block_production_exception need to update controller::failure_is_subjective + FC_DECLARE_DERIVED_EXCEPTION( subjective_block_production_exception, chain_exception, + 3260000, "Subjective exception thrown during block production" ) + + // Objective block validation exceptions are throw out of transaction processing to stop block production + FC_DECLARE_DERIVED_EXCEPTION( objective_block_validation_exception, chain_exception, + 3270000, "Objective exception thrown during block validation" ) + FC_DECLARE_DERIVED_EXCEPTION( disallowed_transaction_extensions_bad_block_exception, objective_block_validation_exception, + 3270001, "Transaction includes disallowed extensions (invalid block)" ) + FC_DECLARE_DERIVED_EXCEPTION( protocol_feature_bad_block_exception, objective_block_validation_exception, + 3270002, "Protocol feature exception (invalid block)" ) + FC_DECLARE_DERIVED_EXCEPTION( pruned_context_free_data_bad_block_exception, objective_block_validation_exception, + 3270003, "Context free data pruned (invalid block)" ) + + FC_DECLARE_DERIVED_EXCEPTION( state_history_exception, chain_exception, + 3280000, "State history exception" ) + } } // eosio::chain diff --git a/libraries/chain/include/eosio/chain/generated_transaction_object.hpp b/libraries/chain/include/eosio/chain/generated_transaction_object.hpp index a93e65296e4..8467f0a99b3 100644 --- a/libraries/chain/include/eosio/chain/generated_transaction_object.hpp +++ b/libraries/chain/include/eosio/chain/generated_transaction_object.hpp @@ -35,9 +35,10 @@ namespace eosio { namespace chain { uint32_t set( const transaction& trx ) { auto trxsize = fc::raw::pack_size( trx ); - packed_trx.resize( trxsize ); - fc::datastream ds( packed_trx.data(), trxsize ); - fc::raw::pack( ds, trx ); + packed_trx.resize_and_fill( trxsize, [&trx](char* data, std::size_t size) { + fc::datastream ds( data, size ); + fc::raw::pack( ds, trx ); + }); return trxsize; } }; diff --git a/libraries/chain/include/eosio/chain/genesis_state.hpp b/libraries/chain/include/eosio/chain/genesis_state.hpp index 5e5b643a1b5..66cb1f3a3a9 100644 --- a/libraries/chain/include/eosio/chain/genesis_state.hpp +++ b/libraries/chain/include/eosio/chain/genesis_state.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -15,7 +16,7 @@ struct genesis_state { static const string eosio_root_key; - chain_config initial_configuration = { + chain_config_v0 initial_configuration = { .max_block_net_usage = config::default_max_block_net_usage, .target_block_net_usage_pct = config::default_target_block_net_usage_pct, .max_transaction_net_usage = config::default_max_transaction_net_usage, @@ -37,6 +38,20 @@ struct genesis_state { .max_authority_depth = config::default_max_auth_depth, }; + static constexpr wasm_config default_initial_wasm_configuration { + .max_mutable_global_bytes = config::default_max_wasm_mutable_global_bytes, + .max_table_elements = config::default_max_wasm_table_elements, + .max_section_elements = config::default_max_wasm_section_elements, + .max_linear_memory_init = config::default_max_wasm_linear_memory_init, + .max_func_local_bytes = config::default_max_wasm_func_local_bytes, + .max_nested_structures = config::default_max_wasm_nested_structures, + .max_symbol_bytes = config::default_max_wasm_symbol_bytes, + .max_module_bytes = config::default_max_wasm_module_bytes, + .max_code_bytes = config::default_max_wasm_code_bytes, + .max_pages = config::default_max_wasm_pages, + .max_call_depth = config::default_max_wasm_call_depth + }; + time_point initial_timestamp; public_key_type initial_key; @@ -58,6 +73,6 @@ struct genesis_state { } } // namespace eosio::chain - +// @swap initial_timestamp initial_key initial_configuration FC_REFLECT(eosio::chain::genesis_state, (initial_timestamp)(initial_key)(initial_configuration)) diff --git a/libraries/chain/include/eosio/chain/global_property_object.hpp b/libraries/chain/include/eosio/chain/global_property_object.hpp index 57d6c745fe2..7b7497e34ec 100644 --- a/libraries/chain/include/eosio/chain/global_property_object.hpp +++ b/libraries/chain/include/eosio/chain/global_property_object.hpp @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include #include #include @@ -24,9 +26,31 @@ namespace eosio { namespace chain { static constexpr uint32_t maximum_version = 2; static_assert(chain_snapshot_header::minimum_compatible_version <= maximum_version, "snapshot_global_property_object_v2 is no longer needed"); - optional proposed_schedule_block_num; + std::optional proposed_schedule_block_num; producer_schedule_type proposed_schedule; - chain_config configuration; + chain_config_v0 configuration; + }; + struct snapshot_global_property_object_v3 { + static constexpr uint32_t minimum_version = 3; + static constexpr uint32_t maximum_version = 3; + static_assert(chain_snapshot_header::minimum_compatible_version <= maximum_version, "snapshot_global_property_object_v3 is no longer needed"); + + std::optional proposed_schedule_block_num; + producer_authority_schedule proposed_schedule; + chain_config_v0 configuration; + chain_id_type chain_id; + }; + struct snapshot_global_property_object_v4 { + static constexpr uint32_t minimum_version = 4; + static constexpr uint32_t maximum_version = 4; + static_assert(chain_snapshot_header::minimum_compatible_version <= maximum_version, "snapshot_global_property_object_v4 is no longer needed"); + + std::optional proposed_schedule_block_num; + producer_authority_schedule proposed_schedule; + chain_config_v0 configuration; + chain_id_type chain_id; + kv_database_config kv_configuration; + wasm_config wasm_configuration; }; } @@ -42,16 +66,38 @@ namespace eosio { namespace chain { public: id_type id; - optional proposed_schedule_block_num; + std::optional proposed_schedule_block_num; shared_producer_authority_schedule proposed_schedule; chain_config configuration; chain_id_type chain_id; + kv_database_config kv_configuration; + wasm_config wasm_configuration; - void initalize_from( const legacy::snapshot_global_property_object_v2& legacy, const chain_id_type& chain_id_val ) { + void initalize_from( const legacy::snapshot_global_property_object_v2& legacy, const chain_id_type& chain_id_val, const kv_database_config& kv_config_val, const wasm_config& wasm_config_val ) { proposed_schedule_block_num = legacy.proposed_schedule_block_num; proposed_schedule = producer_authority_schedule(legacy.proposed_schedule).to_shared(proposed_schedule.producers.get_allocator()); configuration = legacy.configuration; chain_id = chain_id_val; + kv_configuration = kv_config_val; + wasm_configuration = wasm_config_val; + } + + void initalize_from( const legacy::snapshot_global_property_object_v3& legacy, const kv_database_config& kv_config_val, const wasm_config& wasm_config_val ) { + proposed_schedule_block_num = legacy.proposed_schedule_block_num; + proposed_schedule = legacy.proposed_schedule.to_shared(proposed_schedule.producers.get_allocator()); + configuration = legacy.configuration; + chain_id = legacy.chain_id; + kv_configuration = kv_config_val; + wasm_configuration = wasm_config_val; + } + + void initalize_from( const legacy::snapshot_global_property_object_v4& legacy ) { + proposed_schedule_block_num = legacy.proposed_schedule_block_num; + proposed_schedule = legacy.proposed_schedule.to_shared(proposed_schedule.producers.get_allocator()); + configuration = legacy.configuration; + chain_id = legacy.chain_id; + kv_configuration = legacy.kv_configuration; + wasm_configuration = legacy.wasm_configuration; } }; @@ -66,10 +112,12 @@ namespace eosio { namespace chain { >; struct snapshot_global_property_object { - optional proposed_schedule_block_num; + std::optional proposed_schedule_block_num; producer_authority_schedule proposed_schedule; chain_config configuration; chain_id_type chain_id; + kv_database_config kv_configuration; + wasm_config wasm_configuration; }; namespace detail { @@ -79,7 +127,7 @@ namespace eosio { namespace chain { using snapshot_type = snapshot_global_property_object; static snapshot_global_property_object to_snapshot_row( const global_property_object& value, const chainbase::database& ) { - return {value.proposed_schedule_block_num, producer_authority_schedule::from_shared(value.proposed_schedule), value.configuration, value.chain_id}; + return {value.proposed_schedule_block_num, producer_authority_schedule::from_shared(value.proposed_schedule), value.configuration, value.chain_id, value.kv_configuration, value.wasm_configuration}; } static void from_snapshot_row( snapshot_global_property_object&& row, global_property_object& value, chainbase::database& ) { @@ -87,6 +135,8 @@ namespace eosio { namespace chain { value.proposed_schedule = row.proposed_schedule.to_shared(value.proposed_schedule.producers.get_allocator()); value.configuration = row.configuration; value.chain_id = row.chain_id; + value.kv_configuration = row.kv_configuration; + value.wasm_configuration = row.wasm_configuration; } }; } @@ -121,17 +171,25 @@ CHAINBASE_SET_INDEX_TYPE(eosio::chain::dynamic_global_property_object, eosio::chain::dynamic_global_property_multi_index) FC_REFLECT(eosio::chain::global_property_object, - (proposed_schedule_block_num)(proposed_schedule)(configuration)(chain_id) + (proposed_schedule_block_num)(proposed_schedule)(configuration)(chain_id)(kv_configuration)(wasm_configuration) ) FC_REFLECT(eosio::chain::legacy::snapshot_global_property_object_v2, (proposed_schedule_block_num)(proposed_schedule)(configuration) ) -FC_REFLECT(eosio::chain::snapshot_global_property_object, +FC_REFLECT(eosio::chain::legacy::snapshot_global_property_object_v3, (proposed_schedule_block_num)(proposed_schedule)(configuration)(chain_id) ) +FC_REFLECT(eosio::chain::legacy::snapshot_global_property_object_v4, + (proposed_schedule_block_num)(proposed_schedule)(configuration)(chain_id)(kv_configuration)(wasm_configuration) + ) + +FC_REFLECT(eosio::chain::snapshot_global_property_object, + (proposed_schedule_block_num)(proposed_schedule)(configuration)(chain_id)(kv_configuration)(wasm_configuration) + ) + FC_REFLECT(eosio::chain::dynamic_global_property_object, (global_action_sequence) ) diff --git a/libraries/chain/include/eosio/chain/incremental_merkle.hpp b/libraries/chain/include/eosio/chain/incremental_merkle.hpp index 0a84d076f58..f36171d8b89 100644 --- a/libraries/chain/include/eosio/chain/incremental_merkle.hpp +++ b/libraries/chain/include/eosio/chain/incremental_merkle.hpp @@ -247,5 +247,5 @@ typedef incremental_merkle_impl incremental_merkle; typedef incremental_merkle_impl shared_incremental_merkle; } } /// eosio::chain - +// @swap _active_nodes FC_REFLECT( eosio::chain::incremental_merkle, (_active_nodes)(_node_count) ); diff --git a/libraries/chain/include/eosio/chain/kv_chainbase_objects.hpp b/libraries/chain/include/eosio/chain/kv_chainbase_objects.hpp new file mode 100644 index 00000000000..eb285239a22 --- /dev/null +++ b/libraries/chain/include/eosio/chain/kv_chainbase_objects.hpp @@ -0,0 +1,71 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace eosio { namespace chain { + class kv_db_config_object : public chainbase::object { + OBJECT_CTOR(kv_db_config_object) + + id_type id; + backing_store_type backing_store = backing_store_type::CHAINBASE; + }; + + using kv_db_config_index = chainbase::shared_multi_index_container< + kv_db_config_object, + indexed_by, + BOOST_MULTI_INDEX_MEMBER(kv_db_config_object, kv_db_config_object::id_type, id)>>>; + + struct by_kv_key; + + struct kv_object_view { + name contract; + // TODO: Use non-owning memory + fc::blob kv_key; + fc::blob kv_value; + name payer; + }; + + struct kv_object : public chainbase::object { + OBJECT_CTOR(kv_object, (kv_key)(kv_value)) + + static constexpr uint32_t minimum_snapshot_version = 4; + + id_type id; + name contract; + shared_blob kv_key; + shared_blob kv_value; + name payer; + }; + + using kv_index = chainbase::shared_multi_index_container< + kv_object, + indexed_by, member>, + ordered_unique, + composite_key, + member>, + composite_key_compare, unsigned_blob_less>>>>; + +namespace config { + template<> + struct billable_size { + // NOTICE: Do not change any of the constants defined here; otherwise it would cause backward concensus compatibility problem. + static constexpr uint64_t overhead = overhead_per_row_per_index_ram_bytes * 2; + static constexpr uint64_t serialized_shared_blob_size = 8 + 4; // 8 for vector data 4 for vector size + static constexpr uint64_t serialized_kv_object_size_exclude_shared_blobs = 24; // derived from sizeof(id_type) + 2 * sizeof(name) + static constexpr uint64_t value = serialized_kv_object_size_exclude_shared_blobs + serialized_shared_blob_size * 2 + overhead; + }; +} // namespace config + +}} // namespace eosio::chain + +CHAINBASE_SET_INDEX_TYPE(eosio::chain::kv_db_config_object, eosio::chain::kv_db_config_index) +CHAINBASE_SET_INDEX_TYPE(eosio::chain::kv_object, eosio::chain::kv_index) +FC_REFLECT(eosio::chain::kv_db_config_object, (backing_store)) +FC_REFLECT(eosio::chain::kv_object_view, (contract)(kv_key)(kv_value)(payer)) +FC_REFLECT(eosio::chain::kv_object, (contract)(kv_key)(kv_value)(payer)) diff --git a/libraries/chain/include/eosio/chain/kv_config.hpp b/libraries/chain/include/eosio/chain/kv_config.hpp new file mode 100644 index 00000000000..7330fa09acb --- /dev/null +++ b/libraries/chain/include/eosio/chain/kv_config.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include +#include + +namespace eosio { namespace chain { + + /** + * @brief limits for a kv database. + * + * Each database (ram or disk, currently) has its own limits for these parameters. + * The key and value limits apply when adding or modifying elements. They may be reduced + * below existing database entries. + */ + struct kv_database_config { + std::uint32_t max_key_size = 0; ///< the maximum size in bytes of a key + std::uint32_t max_value_size = 0; ///< the maximum size in bytes of a value + std::uint32_t max_iterators = 0; ///< the maximum number of iterators that a contract can have simultaneously. + }; + inline bool operator==(const kv_database_config& lhs, const kv_database_config& rhs) { + return std::tie(lhs.max_key_size, lhs.max_value_size, lhs.max_iterators) + == std::tie(rhs.max_key_size, rhs.max_value_size, rhs.max_iterators); + } + inline bool operator!=(const kv_database_config& lhs, const kv_database_config& rhs) { + return !(lhs == rhs); + } +}} + +FC_REFLECT(eosio::chain::kv_database_config, (max_key_size)(max_value_size)(max_iterators)) + diff --git a/libraries/chain/include/eosio/chain/log_catalog.hpp b/libraries/chain/include/eosio/chain/log_catalog.hpp new file mode 100644 index 00000000000..48f17cd156e --- /dev/null +++ b/libraries/chain/include/eosio/chain/log_catalog.hpp @@ -0,0 +1,275 @@ +#pragma once +#include +#include +#include +#include +#include +#include + +namespace eosio { +namespace chain { + +namespace bfs = boost::filesystem; + +template +void for_each_file_in_dir_matches(const bfs::path& dir, std::string pattern, Lambda&& lambda) { + const std::regex my_filter(pattern); + std::smatch what; + bfs::directory_iterator end_itr; // Default ctor yields past-the-end + for (bfs::directory_iterator p(dir); p != end_itr; ++p) { + // Skip if not a file + if (!bfs::is_regular_file(p->status())) + continue; + // skip if it does not match the pattern + if (!std::regex_match(p->path().filename().string(), what, my_filter)) + continue; + lambda(p->path()); + } +} + +struct null_verifier { + template + void verify(const LogData&, const bfs::path&) {} +}; + +template +struct log_catalog { + using block_num_t = uint32_t; + + struct mapped_type { + block_num_t last_block_num; + bfs::path filename_base; + }; + using collection_t = boost::container::flat_map; + using size_type = typename collection_t::size_type; + static constexpr size_type npos = std::numeric_limits::max(); + + using mapmode = boost::iostreams::mapped_file::mapmode; + + bfs::path retained_dir; + bfs::path archive_dir; + size_type max_retained_files = 10; + collection_t collection; + size_type active_index = npos; + LogData log_data; + LogIndex log_index; + LogVerifier verifier; + + bool empty() const { return collection.empty(); } + + uint32_t first_block_num() const { + if (empty()) + return 0; + return collection.begin()->first; + } + + static bfs::path make_abosolute_dir(const bfs::path& base_dir, bfs::path new_dir) { + if (new_dir.is_relative()) + new_dir = base_dir / new_dir; + + if (!bfs::is_directory(new_dir)) + bfs::create_directories(new_dir); + + return new_dir; + } + + void open(const bfs::path& log_dir, const bfs::path& retained_dir, const bfs::path& archive_dir, const char* name, + const char* suffix_pattern = R"(-\d+-\d+\.log)") { + + this->retained_dir = make_abosolute_dir(log_dir, retained_dir.empty() ? log_dir : retained_dir); + if (!archive_dir.empty()) { + this->archive_dir = make_abosolute_dir(log_dir, archive_dir); + } + + for_each_file_in_dir_matches(this->retained_dir, std::string(name) + suffix_pattern, [this](bfs::path path) { + auto log_path = path; + auto index_path = path.replace_extension("index"); + auto path_without_extension = log_path.parent_path() / log_path.stem().string(); + + LogData log(log_path); + + verifier.verify(log, log_path); + + // check if index file matches the log file + if (!index_matches_data(index_path, log)) + log.construct_index(index_path); + + auto existing_itr = collection.find(log.first_block_num()); + if (existing_itr != collection.end()) { + if (log.last_block_num() <= existing_itr->second.last_block_num) { + wlog("${log_path} contains the overlapping range with ${existing_path}.log, dropping ${log_path} " + "from catalog", + ("log_path", log_path.string())("existing_path", existing_itr->second.filename_base.string())); + return; + } else { + wlog( + "${log_path} contains the overlapping range with ${existing_path}.log, droping ${existing_path}.log " + "from catelog", + ("log_path", log_path.string())("existing_path", existing_itr->second.filename_base.string())); + } + } + + collection.insert_or_assign(log.first_block_num(), mapped_type{log.last_block_num(), path_without_extension}); + }); + } + + bool index_matches_data(const bfs::path& index_path, const LogData& log) const { + if (!bfs::exists(index_path)) + return false; + + auto num_blocks_in_index = bfs::file_size(index_path) / sizeof(uint64_t); + if (num_blocks_in_index != log.num_blocks()) + return false; + + // make sure the last 8 bytes of index and log matches + fc::cfile index_file; + index_file.set_file_path(index_path); + index_file.open("r"); + index_file.seek_end(-sizeof(uint64_t)); + uint64_t pos; + index_file.read(reinterpret_cast(&pos), sizeof(pos)); + return pos == log.last_block_position(); + } + + std::optional get_block_position(uint32_t block_num, mapmode mode = mapmode::readonly) { + try { + if (active_index != npos) { + auto active_item = collection.nth(active_index); + if (active_item->first <= block_num && block_num <= active_item->second.last_block_num && + log_data.flags() == mode) { + return log_index.nth_block_position(block_num - log_data.first_block_num()); + } + } + if (collection.empty() || block_num < collection.begin()->first) + return {}; + + auto it = --collection.upper_bound(block_num); + + if (block_num <= it->second.last_block_num) { + auto name = it->second.filename_base; + log_data.open(name.replace_extension("log"), mode); + log_index.open(name.replace_extension("index")); + this->active_index = collection.index_of(it); + return log_index.nth_block_position(block_num - log_data.first_block_num()); + } + return {}; + } catch (...) { + this->active_index = npos; + return {}; + } + } + + std::pair, uint32_t> ro_stream_for_block(uint32_t block_num) { + auto pos = get_block_position(block_num, mapmode::readonly); + if (pos) { + return log_data.ro_stream_at(*pos); + } + return {fc::datastream(nullptr, 0), static_cast(0)}; + } + + std::pair, uint32_t> rw_stream_for_block(uint32_t block_num) { + auto pos = get_block_position(block_num, mapmode::readwrite); + if (pos) { + return log_data.rw_stream_at(*pos); + } + return {fc::datastream(nullptr, 0), static_cast(0)}; + } + + std::optional id_for_block(uint32_t block_num) { + auto pos = get_block_position(block_num, mapmode::readonly); + if (pos) { + return log_data.block_id_at(*pos); + } + return {}; + } + + static void rename_if_not_exists(bfs::path old_name, bfs::path new_name) { + if (!bfs::exists(new_name)) { + bfs::rename(old_name, new_name); + } else { + bfs::remove(old_name); + wlog("${new_name} already exists, just removing ${old_name}", + ("old_name", old_name.string())("new_name", new_name.string())); + } + } + + static void rename_bundle(bfs::path orig_path, bfs::path new_path) { + rename_if_not_exists(orig_path.replace_extension(".log"), new_path.replace_extension(".log")); + rename_if_not_exists(orig_path.replace_extension(".index"), new_path.replace_extension(".index")); + } + + /// Add a new entry into the catalog. + /// + /// Notice that \c start_block_num must be monotonically increasing between the invocations of this function + /// so that the new entry would be inserted at the end of the flat_map; otherwise, \c active_index would be + /// invalidated and the mapping between the log data their block range would be wrong. This function is only used + /// during the splitting of block log. Using this function for other purpose should make sure if the monotonically + /// increasing block num guarantee can be met. + void add(uint32_t start_block_num, uint32_t end_block_num, const bfs::path& dir, const char* name) { + + const int bufsize = 64; + char buf[bufsize]; + snprintf(buf, bufsize, "%s-%d-%d", name, start_block_num, end_block_num); + bfs::path new_path = retained_dir / buf; + rename_bundle(dir / name, new_path); + + if (this->collection.size() >= max_retained_files) { + const auto items_to_erase = + max_retained_files > 0 ? this->collection.size() - max_retained_files + 1 : this->collection.size(); + for (auto it = this->collection.begin(); it < this->collection.begin() + items_to_erase; ++it) { + auto orig_name = it->second.filename_base; + if (archive_dir.empty()) { + // delete the old files when no backup dir is specified + bfs::remove(orig_name.replace_extension("log")); + bfs::remove(orig_name.replace_extension("index")); + } else { + // move the the archive dir + rename_bundle(orig_name, archive_dir / orig_name.filename()); + } + } + this->collection.erase(this->collection.begin(), this->collection.begin() + items_to_erase); + this->active_index = this->active_index == npos || this->active_index < items_to_erase + ? npos + : this->active_index - items_to_erase; + } + if (max_retained_files > 0) + this->collection.emplace(start_block_num, mapped_type{end_block_num, new_path}); + } + + /// Truncate the catalog so that the log/index bundle containing the block with \c block_num + /// would be rename to \c new_name; the log/index bundles with blocks strictly higher + /// than \c block_num would be deleted, and all the renamed/removed entries would be erased + /// from the catalog. + /// + /// \return if nonzero, it's the starting block number for the log/index bundle being renamed. + uint32_t truncate(uint32_t block_num, bfs::path new_name) { + if (collection.empty()) + return 0; + + auto remove_files = [](typename collection_t::const_reference v) { + auto name = v.second.filename_base; + bfs::remove(name.replace_extension("log")); + bfs::remove(name.replace_extension("index")); + }; + + auto it = collection.upper_bound(block_num); + + if (it == collection.begin() || block_num > (it - 1)->second.last_block_num) { + std::for_each(it, collection.end(), remove_files); + collection.erase(it, collection.end()); + return 0; + } else { + auto truncate_it = --it; + auto name = truncate_it->second.filename_base; + bfs::rename(name.replace_extension("log"), new_name.replace_extension("log")); + bfs::rename(name.replace_extension("index"), new_name.replace_extension("index")); + std::for_each(truncate_it + 1, collection.end(), remove_files); + auto result = truncate_it->first; + collection.erase(truncate_it, collection.end()); + return result; + } + } +}; + +} // namespace chain +} // namespace eosio diff --git a/libraries/chain/include/eosio/chain/log_data_base.hpp b/libraries/chain/include/eosio/chain/log_data_base.hpp new file mode 100644 index 00000000000..c03d2b6f4ce --- /dev/null +++ b/libraries/chain/include/eosio/chain/log_data_base.hpp @@ -0,0 +1,41 @@ +#pragma once +#include + +namespace eosio { +namespace chain { + +template +T read_buffer(const char* buf) { + T result; + memcpy(&result, buf, sizeof(T)); + return result; +} + +template +class log_data_base { + protected: + boost::iostreams::mapped_file file; + + const Derived* self() const { return static_cast(this); } + + public: + using mapmode = boost::iostreams::mapped_file::mapmode; + + log_data_base() = default; + + bool is_open() const { return file.is_open(); } + mapmode flags() const { return file.flags(); } + + const char* data() const { return file.const_data(); } + uint64_t size() const { return file.size(); } + uint32_t last_block_num() const { return self()->block_num_at(last_block_position()); } + uint64_t last_block_position() const { return read_buffer(data() + size() - sizeof(uint64_t)); } + + uint32_t num_blocks() const { + if (self()->first_block_position() == file.size()) + return 0; + return last_block_num() - self()->first_block_num() + 1; + } +}; +} // namespace chain +} // namespace eosio \ No newline at end of file diff --git a/libraries/chain/include/eosio/chain/log_index.hpp b/libraries/chain/include/eosio/chain/log_index.hpp new file mode 100644 index 00000000000..4ff5f33fdde --- /dev/null +++ b/libraries/chain/include/eosio/chain/log_index.hpp @@ -0,0 +1,38 @@ +#pragma once + +#include +#include + +namespace eosio { +namespace chain { + +template +class log_index { + boost::iostreams::mapped_file_source file; + + public: + log_index() = default; + log_index(const boost::filesystem::path& path) { open(path); } + + void open(const boost::filesystem::path& path) { + if (file.is_open()) + file.close(); + file.open(path.generic_string()); + EOS_ASSERT(file.size() % sizeof(uint64_t) == 0, Exception, + "The size of ${file} is not a multiple of sizeof(uint64_t)", ("file", path.generic_string())); + } + + bool is_open() const { return file.is_open(); } + + using iterator = const uint64_t*; + iterator begin() const { return reinterpret_cast(file.data()); } + iterator end() const { return reinterpret_cast(file.data() + file.size()); } + + /// @pre file.size() > 0 + uint64_t back() const { return *(this->end() - 1); } + int num_blocks() const { return file.size() / sizeof(uint64_t); } + uint64_t nth_block_position(uint32_t n) const { return *(begin() + n); } +}; + +} // namespace chain +} // namespace eosio diff --git a/libraries/chain/include/eosio/chain/merkle.hpp b/libraries/chain/include/eosio/chain/merkle.hpp index 00eae073e09..b99d18c1019 100644 --- a/libraries/chain/include/eosio/chain/merkle.hpp +++ b/libraries/chain/include/eosio/chain/merkle.hpp @@ -17,6 +17,6 @@ namespace eosio { namespace chain { /** * Calculates the merkle root of a set of digests, if ids is odd it will duplicate the last id. */ - digest_type merkle( vector ids ); + digest_type merkle( deque ids ); } } /// eosio::chain diff --git a/libraries/chain/include/eosio/chain/name.hpp b/libraries/chain/include/eosio/chain/name.hpp index eef06b81ece..7c20d9685b1 100644 --- a/libraries/chain/include/eosio/chain/name.hpp +++ b/libraries/chain/include/eosio/chain/name.hpp @@ -1,6 +1,8 @@ #pragma once #include #include +#include +#include #include namespace eosio::chain { @@ -13,29 +15,41 @@ namespace fc { } // fc namespace eosio::chain { - static constexpr uint64_t char_to_symbol( char c ) { + constexpr uint64_t char_to_symbol( char c ) { if( c >= 'a' && c <= 'z' ) return (c - 'a') + 6; if( c >= '1' && c <= '5' ) return (c - '1') + 1; + else if( c == '.') + return 0; + else + FC_THROW_EXCEPTION(name_type_exception, "Name contains invalid character: (${c}) ", ("c", std::string(1, c))); + + //unreachable return 0; } - static constexpr uint64_t string_to_uint64_t( std::string_view str ) { + // true if std::string can be converted to name + bool is_string_valid_name(std::string_view str); + + constexpr uint64_t string_to_uint64_t( std::string_view str ) { + EOS_ASSERT(str.size() <= 13, name_type_exception, "Name is longer than 13 characters (${name}) ", ("name", std::string(str))); + uint64_t n = 0; - int i = 0; - for ( ; str[i] && i < 12; ++i) { - // NOTE: char_to_symbol() returns char type, and without this explicit - // expansion to uint64 type, the compilation fails at the point of usage - // of string_to_name(), where the usage requires constant (compile time) expression. - n |= (char_to_symbol(str[i]) & 0x1f) << (64 - 5 * (i + 1)); + int i = (int) str.size(); + if (i >= 13) { + // Only the first 12 characters can be full-range ([.1-5a-z]). + i = 12; + + // The 13th character must be in the range [.1-5a-j] because it needs to be encoded + // using only four bits (64_bits - 5_bits_per_char * 12_chars). + n = char_to_symbol(str[12]); + EOS_ASSERT(n <= 0x0Full, name_type_exception, "invalid 13th character: (${c})", ("c", std::string(1, str[12]))); + } + // Encode full-range characters. + while (--i >= 0) { + n |= char_to_symbol(str[i]) << (64 - 5 * (i + 1)); } - - // The for-loop encoded up to 60 high bits into uint64 'name' variable, - // if (strlen(str) > 12) then encode str[12] into the low (remaining) - // 4 bits of 'name' - if (i == 12) - n |= char_to_symbol(str[12]) & 0x0F; return n; } @@ -81,12 +95,25 @@ namespace eosio::chain { // to its 5-bit slot starting with the highest slot for the first char. // The 13th char, if str is long enough, is encoded into 4-bit chunk // and placed in the lowest 4 bits. 64 = 12 * 5 + 4 - static constexpr name string_to_name( std::string_view str ) + constexpr name string_to_name( std::string_view str ) { return name( string_to_uint64_t( str ) ); } -#define N(X) eosio::chain::string_to_name(#X) + inline namespace literals { +#if defined(__clang__) +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wgnu-string-literal-operator-template" +#endif + template + inline constexpr name operator""_n() { + constexpr const char buf[] = {Str...}; + return name{std::integral_constant::value}; + } +#if defined(__clang__) +# pragma clang diagnostic pop +#endif + } // namespace literals } // eosio::chain diff --git a/libraries/chain/include/eosio/chain/producer_schedule.hpp b/libraries/chain/include/eosio/chain/producer_schedule.hpp index ec0d5bc79dd..849ccce61ee 100644 --- a/libraries/chain/include/eosio/chain/producer_schedule.hpp +++ b/libraries/chain/include/eosio/chain/producer_schedule.hpp @@ -60,7 +60,7 @@ namespace eosio { namespace chain { shared_vector keys; }; - using shared_block_signing_authority = static_variant; + using shared_block_signing_authority = std::variant; struct shared_producer_authority { shared_producer_authority() = delete; @@ -158,7 +158,7 @@ namespace eosio { namespace chain { } }; - using block_signing_authority = static_variant; + using block_signing_authority = std::variant; struct producer_authority { name producer_name; @@ -166,9 +166,9 @@ namespace eosio { namespace chain { template static void for_each_key( const block_signing_authority& authority, Op&& op ) { - authority.visit([&op](const auto &a){ + std::visit([&op](const auto &a){ a.for_each_key(std::forward(op)); - }); + }, authority); } template @@ -177,9 +177,9 @@ namespace eosio { namespace chain { } static std::pair keys_satisfy_and_relevant( const std::set& keys, const block_signing_authority& authority ) { - return authority.visit([&keys](const auto &a){ + return std::visit([&keys](const auto &a){ return a.keys_satisfy_and_relevant(keys); - }); + }, authority); } std::pair keys_satisfy_and_relevant( const std::set& presented_keys ) const { @@ -187,9 +187,9 @@ namespace eosio { namespace chain { } auto to_shared(chainbase::allocator alloc) const { - auto shared_auth = authority.visit([&alloc](const auto& a) { + auto shared_auth = std::visit([&alloc](const auto& a) { return a.to_shared(alloc); - }); + }, authority); return shared_producer_authority(producer_name, std::move(shared_auth)); } @@ -197,11 +197,11 @@ namespace eosio { namespace chain { static auto from_shared( const shared_producer_authority& src ) { producer_authority result; result.producer_name = src.producer_name; - result.authority = src.authority.visit(overloaded { + result.authority = std::visit(overloaded { [](const shared_block_signing_authority_v0& a) { return block_signing_authority_v0::from_shared(a); } - }); + }, src.authority); return result; } @@ -299,6 +299,9 @@ namespace eosio { namespace chain { producer_schedule_change_extension(const producer_schedule_change_extension&) = default; producer_schedule_change_extension( producer_schedule_change_extension&& ) = default; + producer_schedule_change_extension& operator=(const producer_schedule_change_extension&) = default; + producer_schedule_change_extension& operator=(producer_schedule_change_extension&&) = default; + producer_schedule_change_extension( const producer_authority_schedule& sched ) :producer_authority_schedule(sched) {} }; @@ -307,14 +310,14 @@ namespace eosio { namespace chain { inline bool operator == ( const producer_authority& pa, const shared_producer_authority& pb ) { if(pa.producer_name != pb.producer_name) return false; - if(pa.authority.which() != pb.authority.which()) return false; + if(pa.authority.index() != pb.authority.index()) return false; - bool authority_matches = pa.authority.visit([&pb]( const auto& lhs ){ - return pb.authority.visit( [&lhs](const auto& rhs ) { + bool authority_matches = std::visit([&pb]( const auto& lhs ){ + return std::visit( [&lhs](const auto& rhs ) { if (lhs.threshold != rhs.threshold) return false; return std::equal(lhs.keys.cbegin(), lhs.keys.cend(), rhs.keys.cbegin(), rhs.keys.cend()); - }); - }); + }, pb.authority); + }, pa.authority); if (!authority_matches) return false; return true; diff --git a/libraries/chain/include/eosio/chain/protocol_feature_activation.hpp b/libraries/chain/include/eosio/chain/protocol_feature_activation.hpp index 8201b76115e..1f1a49fef56 100644 --- a/libraries/chain/include/eosio/chain/protocol_feature_activation.hpp +++ b/libraries/chain/include/eosio/chain/protocol_feature_activation.hpp @@ -18,6 +18,12 @@ struct protocol_feature_activation : fc::reflect_init { :protocol_features( std::move(pf) ) {} + protocol_feature_activation(const protocol_feature_activation&) = default; + protocol_feature_activation(protocol_feature_activation&&) = default; + + protocol_feature_activation& operator=(protocol_feature_activation&&) = default; + protocol_feature_activation& operator=(const protocol_feature_activation&) = default; + void reflector_init(); vector protocol_features; diff --git a/libraries/chain/include/eosio/chain/protocol_feature_manager.hpp b/libraries/chain/include/eosio/chain/protocol_feature_manager.hpp index 81d22c15b0e..22b0df48905 100644 --- a/libraries/chain/include/eosio/chain/protocol_feature_manager.hpp +++ b/libraries/chain/include/eosio/chain/protocol_feature_manager.hpp @@ -23,6 +23,10 @@ enum class builtin_protocol_feature_t : uint32_t { ram_restrictions, webauthn_key, wtmsig_block_signatures, + action_return_value, + kv_database, + configurable_wasm_limits, + blockchain_parameters }; struct protocol_feature_subjective_restrictions { @@ -90,13 +94,13 @@ class builtin_protocol_feature : public protocol_feature_base { }; struct protocol_feature { - digest_type feature_digest; - digest_type description_digest; - flat_set dependencies; - time_point earliest_allowed_activation_time; - bool preactivation_required = false; - bool enabled = false; - optional builtin_feature; + digest_type feature_digest; + digest_type description_digest; + flat_set dependencies; + time_point earliest_allowed_activation_time; + bool preactivation_required = false; + bool enabled = false; + std::optional builtin_feature; fc::variant to_variant( bool include_subjective_restrictions = true, fc::mutable_variant_object* additional_fields = nullptr )const; @@ -130,7 +134,7 @@ class protocol_feature_set { recognized_t is_recognized( const digest_type& feature_digest, time_point now )const; - optional get_builtin_digest( builtin_protocol_feature_t feature_codename )const; + std::optional get_builtin_digest( builtin_protocol_feature_t feature_codename )const; const protocol_feature& get_protocol_feature( const digest_type& feature_digest )const; @@ -244,7 +248,7 @@ class protocol_feature_set { class protocol_feature_manager { public: - protocol_feature_manager( protocol_feature_set&& pfs ); + protocol_feature_manager( protocol_feature_set&& pfs, std::function get_deep_mind_logger ); class const_iterator : public std::iterator { protected: @@ -313,7 +317,7 @@ class protocol_feature_manager { const protocol_feature_set& get_protocol_feature_set()const { return _protocol_feature_set; } - optional get_builtin_digest( builtin_protocol_feature_t feature_codename )const { + std::optional get_builtin_digest( builtin_protocol_feature_t feature_codename )const { return _protocol_feature_set.get_builtin_digest( feature_codename ); } @@ -364,6 +368,9 @@ class protocol_feature_manager { vector _builtin_protocol_features; size_t _head_of_builtin_activation_list = builtin_protocol_feature_entry::no_previous; bool _initialized = false; + +private: + std::function _get_deep_mind_logger; }; } } // namespace eosio::chain @@ -371,8 +378,10 @@ class protocol_feature_manager { FC_REFLECT(eosio::chain::protocol_feature_subjective_restrictions, (earliest_allowed_activation_time)(preactivation_required)(enabled)) +// @ignore _type FC_REFLECT(eosio::chain::protocol_feature_base, - (protocol_feature_type)(dependencies)(description_digest)(subjective_restrictions)) + (protocol_feature_type)(description_digest)(dependencies)(subjective_restrictions)) +// @ignore _codename FC_REFLECT_DERIVED(eosio::chain::builtin_protocol_feature, (eosio::chain::protocol_feature_base), (builtin_feature_codename)) diff --git a/libraries/chain/include/eosio/chain/protocol_state_object.hpp b/libraries/chain/include/eosio/chain/protocol_state_object.hpp index dc0eabd1f1b..f59bb55f5f0 100644 --- a/libraries/chain/include/eosio/chain/protocol_state_object.hpp +++ b/libraries/chain/include/eosio/chain/protocol_state_object.hpp @@ -85,7 +85,6 @@ FC_REFLECT(eosio::chain::protocol_state_object::activated_protocol_feature, FC_REFLECT(eosio::chain::protocol_state_object, (activated_protocol_features)(preactivated_protocol_features)(whitelisted_intrinsics)(num_supported_key_types) ) - FC_REFLECT(eosio::chain::snapshot_protocol_state_object, (activated_protocol_features)(preactivated_protocol_features)(whitelisted_intrinsics)(num_supported_key_types) ) diff --git a/libraries/chain/include/eosio/chain/resource_limits.hpp b/libraries/chain/include/eosio/chain/resource_limits.hpp index 9ea8521147a..58081614dbc 100644 --- a/libraries/chain/include/eosio/chain/resource_limits.hpp +++ b/libraries/chain/include/eosio/chain/resource_limits.hpp @@ -2,7 +2,9 @@ #include #include #include +#include #include +#include #include #include @@ -23,6 +25,7 @@ namespace eosio { namespace chain { namespace resource_limits { } }; } + struct resource_limits_object; using ratio = impl::ratio; @@ -51,19 +54,22 @@ namespace eosio { namespace chain { namespace resource_limits { int64_t used = 0; ///< quantity used in current window int64_t available = 0; ///< quantity available in current window (based upon fractional reserve) int64_t max = 0; ///< max per window under current congestion + block_timestamp_type last_usage_update_time; ///< last usage timestamp + int64_t current_used = 0; ///< current usage according to the given timestamp }; class resource_limits_manager { public: - explicit resource_limits_manager(chainbase::database& db) - :_db(db) + + explicit resource_limits_manager(chainbase::database& db, std::function get_deep_mind_logger) + :_db(db),_get_deep_mind_logger(get_deep_mind_logger) { } void add_indices(); void initialize_database(); void add_to_snapshot( const snapshot_writer_ptr& snapshot ) const; - void read_from_snapshot( const snapshot_reader_ptr& snapshot ); + void read_from_snapshot( const snapshot_reader_ptr& snapshot, uint32_t version ); void initialize_account( const account_name& account ); void set_block_parameters( const elastic_limit_parameters& cpu_limit_parameters, const elastic_limit_parameters& net_limit_parameters ); @@ -71,7 +77,7 @@ namespace eosio { namespace chain { namespace resource_limits { void update_account_usage( const flat_set& accounts, uint32_t ordinal ); void add_transaction_usage( const flat_set& accounts, uint64_t cpu_usage, uint64_t net_usage, uint32_t ordinal ); - void add_pending_ram_usage( const account_name account, int64_t ram_delta ); + void add_pending_ram_usage( const account_name account, int64_t ram_delta, const storage_usage_trace& trace ); void verify_account_ram_usage( const account_name accunt )const; /// set_account_limits returns true if new ram_bytes limit is more restrictive than the previously set one @@ -93,16 +99,21 @@ namespace eosio { namespace chain { namespace resource_limits { std::pair get_account_cpu_limit( const account_name& name, uint32_t greylist_limit = config::maximum_elastic_resource_multiplier ) const; std::pair get_account_net_limit( const account_name& name, uint32_t greylist_limit = config::maximum_elastic_resource_multiplier ) const; - std::pair get_account_cpu_limit_ex( const account_name& name, uint32_t greylist_limit = config::maximum_elastic_resource_multiplier ) const; - std::pair get_account_net_limit_ex( const account_name& name, uint32_t greylist_limit = config::maximum_elastic_resource_multiplier ) const; + std::pair + get_account_cpu_limit_ex( const account_name& name, uint32_t greylist_limit = config::maximum_elastic_resource_multiplier, const std::optional& current_time={} ) const; + std::pair + get_account_net_limit_ex( const account_name& name, uint32_t greylist_limit = config::maximum_elastic_resource_multiplier, const std::optional& current_time={} ) const; int64_t get_account_ram_usage( const account_name& name ) const; private: + const resource_limits_object& get_account_limits( const account_name& account ) const; + const resource_limits_object& get_or_create_pending_account_limits( const account_name& account ); chainbase::database& _db; + std::function _get_deep_mind_logger; }; } } } /// eosio::chain -FC_REFLECT( eosio::chain::resource_limits::account_resource_limit, (used)(available)(max) ) +FC_REFLECT( eosio::chain::resource_limits::account_resource_limit, (used)(available)(max)(last_usage_update_time)(current_used) ) FC_REFLECT( eosio::chain::resource_limits::ratio, (numerator)(denominator)) FC_REFLECT( eosio::chain::resource_limits::elastic_limit_parameters, (target)(max)(periods)(max_multiplier)(contract_rate)(expand_rate)) diff --git a/libraries/chain/include/eosio/chain/resource_limits_private.hpp b/libraries/chain/include/eosio/chain/resource_limits_private.hpp index e65de0fcdaf..fa19cdd36f4 100644 --- a/libraries/chain/include/eosio/chain/resource_limits_private.hpp +++ b/libraries/chain/include/eosio/chain/resource_limits_private.hpp @@ -2,6 +2,7 @@ #include #include #include +#include #include "multi_index_includes.hpp" @@ -189,6 +190,43 @@ namespace eosio { namespace chain { namespace resource_limits { using usage_accumulator = impl::exponential_moving_average_accumulator<>; + namespace legacy { + struct snapshot_resource_limits_object_v3 { + static constexpr uint32_t minimum_version = 0; + static constexpr uint32_t maximum_version = 3; + static_assert(chain_snapshot_header::minimum_compatible_version <= maximum_version, "snapshot_resource_limits_object_v3 is no longer needed"); + account_name owner; + + int64_t net_weight; + int64_t cpu_weight; + int64_t ram_bytes; + }; + struct snapshot_resource_usage_object_v3 { + static constexpr uint32_t minimum_version = 0; + static constexpr uint32_t maximum_version = 3; + static_assert(chain_snapshot_header::minimum_compatible_version <= maximum_version, "snapshot_resource_usage_object_v3 is no longer needed"); + account_name owner; + usage_accumulator net_usage; + usage_accumulator cpu_usage; + uint64_t ram_usage; + }; + struct snapshot_resource_limits_state_object_v3 { + static constexpr uint32_t minimum_version = 0; + static constexpr uint32_t maximum_version = 3; + static_assert(chain_snapshot_header::minimum_compatible_version <= maximum_version, "snapshot_resource_limits_state_object_v3 is no longer needed"); + + usage_accumulator average_block_net_usage; + usage_accumulator average_block_cpu_usage; + uint64_t pending_net_usage; + uint64_t pending_cpu_usage; + uint64_t total_net_weight; + uint64_t total_cpu_weight; + uint64_t total_ram_bytes; + uint64_t virtual_net_limit; + uint64_t virtual_cpu_limit; + }; + } + /** * Every account that authorizes a transaction is billed for the full size of that transaction. This object * tracks the average usage of that account. @@ -205,6 +243,15 @@ namespace eosio { namespace chain { namespace resource_limits { int64_t cpu_weight = -1; int64_t ram_bytes = -1; + using v3 = legacy::snapshot_resource_limits_object_v3; + void initialize_from( const v3& legacy) { + owner = legacy.owner; + pending = false; + net_weight = legacy.net_weight; + cpu_weight = legacy.cpu_weight; + ram_bytes = legacy.ram_bytes; + } + }; struct by_owner; @@ -233,6 +280,13 @@ namespace eosio { namespace chain { namespace resource_limits { usage_accumulator cpu_usage; uint64_t ram_usage = 0; + using v3 = legacy::snapshot_resource_usage_object_v3; + void initialize_from( const v3& legacy ) { + owner = legacy.owner; + net_usage = legacy.net_usage; + cpu_usage = legacy.cpu_usage; + ram_usage = legacy.ram_usage; + } }; using resource_usage_index = chainbase::shared_multi_index_container< @@ -259,6 +313,8 @@ namespace eosio { namespace chain { namespace resource_limits { uint32_t account_cpu_usage_average_window = config::account_cpu_usage_average_window_ms / config::block_interval_ms; uint32_t account_net_usage_average_window = config::account_net_usage_average_window_ms / config::block_interval_ms; + + using v3 = resource_limits_config_object; }; using resource_limits_config_index = chainbase::shared_multi_index_container< @@ -314,6 +370,19 @@ namespace eosio { namespace chain { namespace resource_limits { */ uint64_t virtual_cpu_limit = 0ULL; + using v3 = legacy::snapshot_resource_limits_state_object_v3; + void initialize_from( const v3& legacy ) { + average_block_net_usage = legacy.average_block_net_usage; + average_block_cpu_usage = legacy.average_block_cpu_usage; + pending_net_usage = legacy.pending_net_usage; + pending_cpu_usage = legacy.pending_cpu_usage; + total_net_weight = legacy.total_net_weight; + total_cpu_weight = legacy.total_cpu_weight; + total_ram_bytes = legacy.total_ram_bytes; + virtual_net_limit = legacy.virtual_net_limit; + virtual_cpu_limit = legacy.virtual_cpu_limit; + } + }; using resource_limits_state_index = chainbase::shared_multi_index_container< @@ -337,3 +406,7 @@ FC_REFLECT(eosio::chain::resource_limits::resource_limits_object, (owner)(net_we FC_REFLECT(eosio::chain::resource_limits::resource_usage_object, (owner)(net_usage)(cpu_usage)(ram_usage)) FC_REFLECT(eosio::chain::resource_limits::resource_limits_config_object, (cpu_limit_parameters)(net_limit_parameters)(account_cpu_usage_average_window)(account_net_usage_average_window)) FC_REFLECT(eosio::chain::resource_limits::resource_limits_state_object, (average_block_net_usage)(average_block_cpu_usage)(pending_net_usage)(pending_cpu_usage)(total_net_weight)(total_cpu_weight)(total_ram_bytes)(virtual_net_limit)(virtual_cpu_limit)) + +FC_REFLECT(eosio::chain::resource_limits::legacy::snapshot_resource_limits_object_v3, (owner)(net_weight)(cpu_weight)(ram_bytes)) +FC_REFLECT(eosio::chain::resource_limits::legacy::snapshot_resource_usage_object_v3, (owner)(net_usage)(cpu_usage)(ram_usage)) +FC_REFLECT(eosio::chain::resource_limits::legacy::snapshot_resource_limits_state_object_v3, (average_block_net_usage)(average_block_cpu_usage)(pending_net_usage)(pending_cpu_usage)(total_net_weight)(total_cpu_weight)(total_ram_bytes)(virtual_net_limit)(virtual_cpu_limit)) diff --git a/libraries/chain/include/eosio/chain/reversible_block_object.hpp b/libraries/chain/include/eosio/chain/reversible_block_object.hpp index 690d3c1953c..927fa39fc57 100644 --- a/libraries/chain/include/eosio/chain/reversible_block_object.hpp +++ b/libraries/chain/include/eosio/chain/reversible_block_object.hpp @@ -16,9 +16,10 @@ namespace eosio { namespace chain { shared_string packedblock; void set_block( const signed_block_ptr& b ) { - packedblock.resize( fc::raw::pack_size( *b ) ); - fc::datastream ds( packedblock.data(), packedblock.size() ); - fc::raw::pack( ds, *b ); + packedblock.resize_and_fill( fc::raw::pack_size( *b ), [&b](char* data, std::size_t size) { + fc::datastream ds( data, size ); + fc::raw::pack( ds, *b ); + }); } signed_block_ptr get_block()const { @@ -34,7 +35,7 @@ namespace eosio { namespace chain { fc::raw::unpack( ds, h ); // Only need the block id to then look up the block state in fork database, so just unpack the block_header from the stored packed data. // Avoid calling get_block() since that constructs a new signed_block in heap memory and unpacks the full signed_block from the stored packed data. - return h.id(); + return h.calculate_id(); } }; diff --git a/libraries/chain/include/eosio/chain/snapshot.hpp b/libraries/chain/include/eosio/chain/snapshot.hpp index 69f89987912..e5bfd262439 100644 --- a/libraries/chain/include/eosio/chain/snapshot.hpp +++ b/libraries/chain/include/eosio/chain/snapshot.hpp @@ -71,7 +71,7 @@ namespace eosio { namespace chain { struct abstract_snapshot_row_writer { virtual void write(ostream_wrapper& out) const = 0; virtual void write(fc::sha256::encoder& out) const = 0; - virtual variant to_variant() const = 0; + virtual fc::variant to_variant() const = 0; virtual std::string row_type_name() const = 0; }; @@ -94,7 +94,7 @@ namespace eosio { namespace chain { } fc::variant to_variant() const override { - variant var; + fc::variant var; fc::to_variant(data, var); return var; } diff --git a/libraries/chain/include/eosio/chain/symbol.hpp b/libraries/chain/include/eosio/chain/symbol.hpp index 1faa3038921..071505d83d3 100644 --- a/libraries/chain/include/eosio/chain/symbol.hpp +++ b/libraries/chain/include/eosio/chain/symbol.hpp @@ -165,6 +165,25 @@ namespace eosio { return lhs.value() > rhs.value(); } + inline bool operator== (const extended_symbol& lhs, const extended_symbol& rhs) + { + return std::tie(lhs.sym, lhs.contract) == std::tie(rhs.sym, rhs.contract); + } + + inline bool operator!= (const extended_symbol& lhs, const extended_symbol& rhs) + { + return std::tie(lhs.sym, lhs.contract) != std::tie(rhs.sym, rhs.contract); + } + + inline bool operator< (const extended_symbol& lhs, const extended_symbol& rhs) + { + return std::tie(lhs.sym, lhs.contract) < std::tie(rhs.sym, rhs.contract); + } + + inline bool operator> (const extended_symbol& lhs, const extended_symbol& rhs) + { + return std::tie(lhs.sym, lhs.contract) > std::tie(rhs.sym, rhs.contract); + } } // namespace chain } // namespace eosio diff --git a/libraries/chain/include/eosio/chain/thread_utils.hpp b/libraries/chain/include/eosio/chain/thread_utils.hpp index 3ec462d5402..077b14a4fb6 100644 --- a/libraries/chain/include/eosio/chain/thread_utils.hpp +++ b/libraries/chain/include/eosio/chain/thread_utils.hpp @@ -1,11 +1,11 @@ #pragma once -#include #include #include #include #include #include +#include namespace eosio { namespace chain { @@ -32,7 +32,7 @@ namespace eosio { namespace chain { boost::asio::thread_pool _thread_pool; boost::asio::io_context _ioc; - fc::optional _ioc_work; + std::optional _ioc_work; }; diff --git a/libraries/chain/include/eosio/chain/trace.hpp b/libraries/chain/include/eosio/chain/trace.hpp index b617baf68b4..f8e20533067 100644 --- a/libraries/chain/include/eosio/chain/trace.hpp +++ b/libraries/chain/include/eosio/chain/trace.hpp @@ -31,7 +31,7 @@ namespace eosio { namespace chain { fc::unsigned_int action_ordinal; fc::unsigned_int creator_action_ordinal; fc::unsigned_int closest_unnotified_ancestor_action_ordinal; - fc::optional receipt; + std::optional receipt; action_name receiver; action act; bool context_free = false; @@ -40,27 +40,29 @@ namespace eosio { namespace chain { transaction_id_type trx_id; ///< the transaction that generated this action uint32_t block_num = 0; block_timestamp_type block_time; - fc::optional producer_block_id; + std::optional producer_block_id; flat_set account_ram_deltas; - fc::optional except; - fc::optional error_code; + flat_set account_disk_deltas; + std::optional except; + std::optional error_code; + std::vector return_value; }; struct transaction_trace { transaction_id_type id; uint32_t block_num = 0; block_timestamp_type block_time; - fc::optional producer_block_id; - fc::optional receipt; + std::optional producer_block_id; + std::optional receipt; fc::microseconds elapsed; uint64_t net_usage = 0; bool scheduled = false; vector action_traces; - fc::optional account_ram_delta; + std::optional account_ram_delta; transaction_trace_ptr failed_dtrx_trace; - fc::optional except; - fc::optional error_code; + std::optional except; + std::optional error_code; std::exception_ptr except_ptr; }; @@ -71,7 +73,7 @@ namespace eosio { namespace chain { if (tt.action_traces.empty()) return false; const auto& act = tt.action_traces[0].act; - if (act.account != eosio::chain::config::system_account_name || act.name != N(onblock) || + if (act.account != eosio::chain::config::system_account_name || act.name != "onblock"_n || act.authorization.size() != 1) return false; const auto& auth = act.authorization[0]; @@ -79,6 +81,36 @@ namespace eosio { namespace chain { auth.permission == eosio::chain::config::active_name; } + #define STORAGE_EVENT_ID( FORMAT, ... ) \ + fc::format_string( FORMAT, fc::mutable_variant_object()__VA_ARGS__ ) + + struct storage_usage_trace { + public: + storage_usage_trace(uint32_t action_id, std::string event_id, const char* family, const char* operation) + :storage_usage_trace(action_id, std::move(event_id), family, operation, ".") + {} + + storage_usage_trace(uint32_t action_id, std::string&& event_id, const char* family, const char* operation, const char* legacy_tag) + :action_id(action_id),event_id(std::move(event_id)),family(family),operation(operation),legacy_tag(legacy_tag) + {} + + uint32_t action_id = 0; + const std::string event_id = "generic"; + const char* family = "generic"; + const char* operation = "generic"; + const char* legacy_tag = "generic"; + + private: + storage_usage_trace(uint32_t action_id) + :action_id(action_id) + {} + + friend storage_usage_trace generic_storage_usage_trace(uint32_t); + }; + + inline storage_usage_trace generic_storage_usage_trace(uint32_t action_id) { + return {action_id}; + } } } /// namespace eosio::chain FC_REFLECT( eosio::chain::account_delta, @@ -87,8 +119,9 @@ FC_REFLECT( eosio::chain::account_delta, FC_REFLECT( eosio::chain::action_trace, (action_ordinal)(creator_action_ordinal)(closest_unnotified_ancestor_action_ordinal)(receipt) (receiver)(act)(context_free)(elapsed)(console)(trx_id)(block_num)(block_time) - (producer_block_id)(account_ram_deltas)(except)(error_code) ) + (producer_block_id)(account_ram_deltas)(account_disk_deltas)(except)(error_code)(return_value) ) +// @ignore except_ptr FC_REFLECT( eosio::chain::transaction_trace, (id)(block_num)(block_time)(producer_block_id) (receipt)(elapsed)(net_usage)(scheduled) (action_traces)(account_ram_delta)(failed_dtrx_trace)(except)(error_code) ) diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index 39c7c965324..662d5264d61 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -27,7 +27,7 @@ namespace eosio { namespace chain { namespace detail { template struct transaction_extension_types { - using transaction_extension_t = fc::static_variant< Ts... >; + using transaction_extension_t = std::variant< Ts... >; using decompose_t = decompose< Ts... >; }; } @@ -49,12 +49,6 @@ namespace eosio { namespace chain { * with a block_header::timestamp greater than expiration is * deemed irreversible, then a user can safely trust the transaction * will never be included. - * - - * Each region is an independent blockchain, it is included as routing - * information for inter-blockchain communication. A contract in this - * region might generate or authorize a transaction intended for a foreign - * region. */ struct transaction_header { time_point_sec expiration; ///< the time at which a transaction expires @@ -64,23 +58,23 @@ namespace eosio { namespace chain { uint8_t max_cpu_usage_ms = 0; /// upper limit on the total CPU time billed for this transaction fc::unsigned_int delay_sec = 0UL; /// number of seconds to delay this transaction for during which it may be canceled. - /** - * @return the absolute block number given the relative ref_block_num - */ - block_num_type get_ref_blocknum( block_num_type head_blocknum )const { - return ((head_blocknum/0xffff)*0xffff) + head_blocknum%0xffff; - } void set_reference_block( const block_id_type& reference_block ); bool verify_reference_block( const block_id_type& reference_block )const; void validate()const; }; /** - * A transaction consits of a set of messages which must all be applied or + * A transaction consists of a set of messages which must all be applied or * all are rejected. These messages have access to data within the given * read and write scopes. */ struct transaction : public transaction_header { + transaction() = default; + explicit transaction( const transaction& ) = default; + transaction( transaction&& ) = default; + transaction& operator=(const transaction&) = delete; + transaction& operator=(transaction&&) = default; + vector context_free_actions; vector actions; extensions_type transaction_extensions; @@ -110,16 +104,13 @@ namespace eosio { namespace chain { struct signed_transaction : public transaction { signed_transaction() = default; -// signed_transaction( const signed_transaction& ) = default; -// signed_transaction( signed_transaction&& ) = default; - signed_transaction( transaction&& trx, const vector& signatures, const vector& context_free_data) + explicit signed_transaction( const signed_transaction& ) = default; + signed_transaction( signed_transaction&& ) = default; + signed_transaction& operator=(const signed_transaction&) = delete; + signed_transaction& operator=(signed_transaction&&) = default; + signed_transaction( transaction trx, vector signatures, vector context_free_data) : transaction(std::move(trx)) - , signatures(signatures) - , context_free_data(context_free_data) - {} - signed_transaction( transaction&& trx, const vector& signatures, vector&& context_free_data) - : transaction(std::move(trx)) - , signatures(signatures) + , signatures(std::move(signatures)) , context_free_data(std::move(context_free_data)) {} @@ -133,42 +124,38 @@ namespace eosio { namespace chain { bool allow_duplicate_keys = false )const; }; - struct packed_transaction : fc::reflect_init { + struct packed_transaction_v0 : fc::reflect_init { enum class compression_type { none = 0, zlib = 1, }; - packed_transaction() = default; - packed_transaction(packed_transaction&&) = default; - explicit packed_transaction(const packed_transaction&) = default; - packed_transaction& operator=(const packed_transaction&) = delete; - packed_transaction& operator=(packed_transaction&&) = default; + packed_transaction_v0() = default; + packed_transaction_v0(packed_transaction_v0&&) = default; + explicit packed_transaction_v0(const packed_transaction_v0&) = default; + packed_transaction_v0& operator=(const packed_transaction_v0&) = delete; + packed_transaction_v0& operator=(packed_transaction_v0&&) = default; - explicit packed_transaction(const signed_transaction& t, compression_type _compression = compression_type::none) + explicit packed_transaction_v0(const signed_transaction& t, compression_type _compression = compression_type::none) :signatures(t.signatures), compression(_compression), unpacked_trx(t), trx_id(unpacked_trx.id()) { local_pack_transaction(); local_pack_context_free_data(); } - explicit packed_transaction(signed_transaction&& t, compression_type _compression = compression_type::none) + explicit packed_transaction_v0(signed_transaction&& t, compression_type _compression = compression_type::none) :signatures(t.signatures), compression(_compression), unpacked_trx(std::move(t)), trx_id(unpacked_trx.id()) { local_pack_transaction(); local_pack_context_free_data(); } - // used by abi_serializer - packed_transaction( bytes&& packed_txn, vector&& sigs, bytes&& packed_cfd, compression_type _compression ); - packed_transaction( bytes&& packed_txn, vector&& sigs, vector&& cfd, compression_type _compression ); - packed_transaction( transaction&& t, vector&& sigs, bytes&& packed_cfd, compression_type _compression ); + packed_transaction_v0(const bytes& packed_txn, const vector& sigs, const bytes& packed_cfd, compression_type _compression); - friend bool operator==(const packed_transaction& lhs, const packed_transaction& rhs) { - return std::tie(lhs.signatures, lhs.compression, lhs.packed_context_free_data, lhs.packed_trx) == - std::tie(rhs.signatures, rhs.compression, rhs.packed_context_free_data, rhs.packed_trx); - } - friend bool operator!=(const packed_transaction& lhs, const packed_transaction& rhs) { return !(lhs == rhs); } + // used by abi_serializer + packed_transaction_v0( bytes&& packed_txn, vector&& sigs, bytes&& packed_cfd, compression_type _compression ); + packed_transaction_v0( bytes&& packed_txn, vector&& sigs, vector&& cfd, compression_type _compression ); + packed_transaction_v0( transaction&& t, vector&& sigs, bytes&& packed_cfd, compression_type _compression ); uint32_t get_unprunable_size()const; uint32_t get_prunable_size()const; @@ -176,7 +163,6 @@ namespace eosio { namespace chain { digest_type packed_digest()const; const transaction_id_type& id()const { return trx_id; } - bytes get_raw_transaction()const; time_point_sec expiration()const { return unpacked_trx.expiration; } const vector& get_context_free_data()const { return unpacked_trx.context_free_data; } @@ -193,23 +179,173 @@ namespace eosio { namespace chain { void local_pack_transaction(); void local_pack_context_free_data(); + friend struct fc::reflector; + friend struct fc::reflector_init_visitor; + friend struct fc::has_reflector_init; + friend struct packed_transaction; + void reflector_init(); + private: + vector signatures; + fc::enum_type compression; + bytes packed_context_free_data; + bytes packed_trx; + + private: + // cache unpacked trx, for thread safety do not modify after construction + signed_transaction unpacked_trx; + transaction_id_type trx_id; + }; + + using packed_transaction_v0_ptr = std::shared_ptr; + + struct packed_transaction : fc::reflect_init { + + struct prunable_data_type { + enum class compression_type : uint8_t { + none = 0, + zlib = 1, + COMPRESSION_TYPE_COUNT + }; + // Do not exceed 127 as that will break compatibility in serialization format + static_assert( static_cast(compression_type::COMPRESSION_TYPE_COUNT) <= 127 ); + + struct none { + digest_type digest; + + digest_type prunable_digest() const; + + friend bool operator==(const none& lhs, const none& rhs) { + return lhs.digest == rhs.digest; + } + friend bool operator!=(const none& lhs, const none& rhs) { return !(lhs == rhs); } + }; + + using segment_type = std::variant; + + struct partial { + std::vector signatures; + std::vector context_free_segments; + + digest_type prunable_digest() const; + + friend bool operator==(const partial& lhs, const partial& rhs) { + return std::tie( lhs.signatures, lhs.context_free_segments ) == + std::tie( rhs.signatures, rhs.context_free_segments ); + } + friend bool operator!=(const partial& lhs, const partial& rhs) { return !(lhs == rhs); } + }; + + struct full { + std::vector signatures; + std::vector context_free_segments; + + digest_type prunable_digest() const; + + friend bool operator==(const full& lhs, const full& rhs) { + return std::tie( lhs.signatures, lhs.context_free_segments ) == + std::tie( rhs.signatures, rhs.context_free_segments ); + } + friend bool operator!=(const full& lhs, const full& rhs) { return !(lhs == rhs); } + }; + + struct full_legacy { + std::vector signatures; + bytes packed_context_free_data; + vector context_free_segments; + + digest_type prunable_digest() const; + + friend bool operator==(const full_legacy& lhs, const full_legacy& rhs) { + return std::tie( lhs.signatures, lhs.packed_context_free_data, lhs.context_free_segments ) == + std::tie( rhs.signatures, rhs.packed_context_free_data, rhs.context_free_segments ); + } + friend bool operator!=(const full_legacy& lhs, const full_legacy& rhs) { return !(lhs == rhs); } + }; + + using prunable_data_t = std::variant< full_legacy, + none, + partial, + full >; + + friend bool operator==(const prunable_data_type& lhs, const prunable_data_type& rhs) { + return lhs.prunable_data == rhs.prunable_data; + } + friend bool operator!=(const prunable_data_type& lhs, const prunable_data_type& rhs) { return !(lhs == rhs); } + + prunable_data_type prune_all() const; + digest_type digest() const; + + // Returns the maximum pack size of any prunable_data that is reachable + // by pruning this object. + std::size_t maximum_pruned_pack_size( compression_type segment_compression ) const; + + prunable_data_t prunable_data; + }; + + using compression_type = packed_transaction_v0::compression_type; + using cf_compression_type = prunable_data_type::compression_type; + + packed_transaction() = default; + packed_transaction(packed_transaction&&) = default; + explicit packed_transaction(const packed_transaction&) = default; + packed_transaction& operator=(const packed_transaction&) = delete; + packed_transaction& operator=(packed_transaction&&) = default; + + packed_transaction(const packed_transaction_v0& other, bool legacy); + packed_transaction(packed_transaction_v0&& other, bool legacy); + explicit packed_transaction(signed_transaction t, bool legacy, compression_type _compression = compression_type::none); + + packed_transaction_v0_ptr to_packed_transaction_v0() const; + + friend bool operator==(const packed_transaction& lhs, const packed_transaction& rhs) { + return std::tie(lhs.compression, lhs.prunable_data, lhs.packed_trx) == + std::tie(rhs.compression, rhs.prunable_data, rhs.packed_trx); + } + friend bool operator!=(const packed_transaction& lhs, const packed_transaction& rhs) { return !(lhs == rhs); } + + uint32_t get_unprunable_size()const; + uint32_t get_prunable_size()const; + uint32_t get_estimated_size()const { return estimated_size; } + + digest_type packed_digest()const; + + const transaction_id_type& id()const { return trx_id; } + + time_point_sec expiration()const { return unpacked_trx.expiration; } + const transaction& get_transaction()const { return unpacked_trx; } + // Returns nullptr if the signatures were pruned + const vector* get_signatures()const; + // Returns nullptr if any context_free_data segment was pruned + const vector* get_context_free_data()const; + // Returns nullptr if the context_free_data segment was pruned or segment_ordinal is out of range. + const bytes* get_context_free_data(std::size_t segment_ordinal)const; + const fc::enum_type& get_compression()const { return compression; } + const bytes& get_packed_transaction()const { return packed_trx; } + const packed_transaction::prunable_data_type& get_prunable_data() const { return prunable_data; } + + void prune_all(); + + std::size_t maximum_pruned_pack_size( cf_compression_type segment_compression ) const; + + private: friend struct fc::reflector; friend struct fc::reflector_init_visitor; friend struct fc::has_reflector_init; void reflector_init(); + uint32_t calculate_estimated_size() const; private: - vector signatures; + uint32_t estimated_size = 0; fc::enum_type compression; - bytes packed_context_free_data; - bytes packed_trx; + prunable_data_type prunable_data; + bytes packed_trx; // packed and compressed (according to compression) transaction private: - // cache unpacked trx, for thread safety do not modify after construction - signed_transaction unpacked_trx; + // cache unpacked trx, for thread safety do not modify any attributes after construction + transaction unpacked_trx; transaction_id_type trx_id; }; - using packed_transaction_ptr = std::shared_ptr; + using packed_transaction_ptr = std::shared_ptr; uint128_t transaction_id_to_sender_id( const transaction_id_type& tid ); @@ -220,6 +356,14 @@ FC_REFLECT( eosio::chain::transaction_header, (expiration)(ref_block_num)(ref_bl (max_net_usage_words)(max_cpu_usage_ms)(delay_sec) ) FC_REFLECT_DERIVED( eosio::chain::transaction, (eosio::chain::transaction_header), (context_free_actions)(actions)(transaction_extensions) ) FC_REFLECT_DERIVED( eosio::chain::signed_transaction, (eosio::chain::transaction), (signatures)(context_free_data) ) -FC_REFLECT_ENUM( eosio::chain::packed_transaction::compression_type, (none)(zlib)) -// @ignore unpacked_trx -FC_REFLECT( eosio::chain::packed_transaction, (signatures)(compression)(packed_context_free_data)(packed_trx) ) +FC_REFLECT_ENUM( eosio::chain::packed_transaction_v0::compression_type, (none)(zlib)) +// @ignore unpacked_trx trx_id +FC_REFLECT( eosio::chain::packed_transaction_v0, (signatures)(compression)(packed_context_free_data)(packed_trx) ) +// @ignore estimated_size unpacked_trx trx_id +FC_REFLECT( eosio::chain::packed_transaction, (compression)(prunable_data)(packed_trx) ) +FC_REFLECT( eosio::chain::packed_transaction::prunable_data_type, (prunable_data)); +FC_REFLECT( eosio::chain::packed_transaction::prunable_data_type::none, (digest)) +FC_REFLECT( eosio::chain::packed_transaction::prunable_data_type::partial, (signatures)( context_free_segments)) +FC_REFLECT( eosio::chain::packed_transaction::prunable_data_type::full, (signatures)( context_free_segments)) +// @ignore context_free_segments +FC_REFLECT( eosio::chain::packed_transaction::prunable_data_type::full_legacy, (signatures)( packed_context_free_data)) diff --git a/libraries/chain/include/eosio/chain/transaction_context.hpp b/libraries/chain/include/eosio/chain/transaction_context.hpp index 4f3c10442f6..29dd50ae0b6 100644 --- a/libraries/chain/include/eosio/chain/transaction_context.hpp +++ b/libraries/chain/include/eosio/chain/transaction_context.hpp @@ -1,4 +1,5 @@ #pragma once +#include #include #include #include @@ -6,6 +7,22 @@ namespace eosio { namespace chain { + /** + * This is a little helper class used by deep mind tracer + * to record on-going execution id/index of all actions executed + * by a given transaction. + */ + class action_id_type { + public: + action_id_type(): id(0) {} + + inline void increment() { id++; } + inline uint32_t current() const { return id; } + + private: + uint32_t id; + }; + struct transaction_checktime_timer { public: transaction_checktime_timer() = delete; @@ -31,13 +48,14 @@ namespace eosio { namespace chain { class transaction_context { private: - void init( uint64_t initial_net_usage); + void init( uint64_t initial_net_usage ); + + void init_for_input_trx_common( uint64_t initial_net_usage, bool skip_recording ); public: transaction_context( controller& c, - const signed_transaction& t, - const transaction_id_type& trx_id, + const packed_transaction& t, transaction_checktime_timer&& timer, fc::time_point start = fc::time_point::now() ); @@ -45,7 +63,10 @@ namespace eosio { namespace chain { void init_for_input_trx( uint64_t packed_trx_unprunable_size, uint64_t packed_trx_prunable_size, - bool skip_recording); + bool skip_recording ); + + void init_for_input_trx_with_explicit_net( uint32_t explicit_net_usage_words, + bool skip_recording ); void init_for_deferred_trx( fc::time_point published ); @@ -54,12 +75,36 @@ namespace eosio { namespace chain { void squash(); void undo(); - inline void add_net_usage( uint64_t u ) { net_usage += u; check_net_usage(); } + inline void add_net_usage( uint64_t u ) { + if( explicit_net_usage ) return; + net_usage += u; + check_net_usage(); + } + + inline void round_up_net_usage() { + if( explicit_net_usage ) return; + net_usage = ((net_usage + 7)/8)*8; // Round up to nearest multiple of word size (8 bytes) + check_net_usage(); + } void check_net_usage()const; void checktime()const; + template + inline DigestType hash_with_checktime( const char* data, uint32_t datalen )const { + const size_t bs = eosio::chain::config::hashing_checktime_block_size; + typename DigestType::encoder enc; + while ( datalen > bs ) { + enc.write( data, bs ); + data += bs; + datalen -= bs; + checktime(); + } + enc.write( data, datalen ); + return enc.result(); + } + void pause_billing_timer(); void resume_billing_timer(); @@ -69,12 +114,14 @@ namespace eosio { namespace chain { void validate_referenced_accounts( const transaction& trx, bool enforce_actor_whitelist_blacklist )const; + uint32_t get_action_id() const { return action_id.current(); } + private: friend struct controller_impl; friend class apply_context; - void add_ram_usage( account_name account, int64_t ram_delta ); + void add_ram_usage( account_name account, int64_t ram_delta, const storage_usage_trace& trace ); action_trace& get_action_trace( uint32_t action_ordinal ); const action_trace& get_action_trace( uint32_t action_ordinal )const; @@ -106,18 +153,18 @@ namespace eosio { namespace chain { public: controller& control; - const signed_transaction& trx; - transaction_id_type id; - optional undo_session; + const packed_transaction& packed_trx; + combined_session undo_session; transaction_trace_ptr trace; fc::time_point start; fc::time_point published; - vector executed; + deque executed_action_receipt_digests; flat_set bill_to_accounts; flat_set validate_ram_usage; + flat_set validate_disk_usage; /// the maximum number of virtual CPU instructions of the transaction that can be safely billed to the billable accounts uint64_t initial_max_billable_cpu = 0; @@ -135,6 +182,9 @@ namespace eosio { namespace chain { transaction_checktime_timer transaction_timer; + /// kept to track ids of action_traces push via this transaction + action_id_type action_id; + private: bool is_initialized = false; @@ -144,6 +194,7 @@ namespace eosio { namespace chain { bool net_limit_due_to_greylist = false; uint64_t eager_net_limit = 0; uint64_t& net_usage; /// reference to trace->net_usage + bool explicit_net_usage = false; bool cpu_limit_due_to_greylist = false; diff --git a/libraries/chain/include/eosio/chain/transaction_metadata.hpp b/libraries/chain/include/eosio/chain/transaction_metadata.hpp index bb48c71e9b4..2508c602de3 100644 --- a/libraries/chain/include/eosio/chain/transaction_metadata.hpp +++ b/libraries/chain/include/eosio/chain/transaction_metadata.hpp @@ -40,10 +40,13 @@ class transaction_metadata { private: struct private_type{}; - static void check_variable_sig_size(const packed_transaction_ptr& trx, uint32_t max) { - for(const signature_type& sig : trx->get_signed_transaction().signatures) + static const vector& check_variable_sig_size(const packed_transaction_ptr& trx, uint32_t max) { + const vector* sigs = trx->get_signatures(); + EOS_ASSERT( sigs, tx_no_signature, "signatures pruned from packed_transaction" ); + for(const signature_type& sig : *sigs) EOS_ASSERT(sig.variable_size() <= max, sig_variable_size_limit_exception, "signature variable length component size (${s}) greater than subjective maximum (${m})", ("s", sig.variable_size())("m", max)); + return *sigs; } public: @@ -69,6 +72,7 @@ class transaction_metadata { const transaction_id_type& id()const { return _packed_trx->id(); } fc::microseconds signature_cpu_usage()const { return _sig_cpu_usage; } const flat_set& recovered_keys()const { return _recovered_pub_keys; } + uint32_t get_estimated_size() const; /// Thread safe. /// @returns transaction_metadata_ptr or exception via future @@ -79,10 +83,9 @@ class transaction_metadata { /// @returns constructed transaction_metadata with no key recovery (sig_cpu_usage=0, recovered_pub_keys=empty) static transaction_metadata_ptr - create_no_recover_keys( const packed_transaction& trx, trx_type t ) { - return std::make_shared( private_type(), - std::make_shared( trx ), fc::microseconds(), flat_set(), - t == trx_type::implicit, t == trx_type::scheduled ); + create_no_recover_keys( packed_transaction_ptr trx, trx_type t ) { + return std::make_shared( private_type(), std::move(trx), + fc::microseconds(), flat_set(), t == trx_type::implicit, t == trx_type::scheduled ); } }; diff --git a/libraries/chain/include/eosio/chain/types.hpp b/libraries/chain/include/eosio/chain/types.hpp index 6f259f73b02..15d36fb5f81 100644 --- a/libraries/chain/include/eosio/chain/types.hpp +++ b/libraries/chain/include/eosio/chain/types.hpp @@ -8,17 +8,18 @@ #include #include #include -#include #include #include #include #include #include -#include #include #include #include +#include +#include + #include #include #include @@ -47,7 +48,6 @@ namespace eosio { namespace chain { using std::vector; using std::unordered_map; using std::string; - using std::deque; using std::shared_ptr; using std::weak_ptr; using std::unique_ptr; @@ -62,11 +62,9 @@ namespace eosio { namespace chain { using std::all_of; using fc::path; - using fc::smart_ref; using fc::variant_object; using fc::variant; using fc::enum_type; - using fc::optional; using fc::unsigned_int; using fc::signed_int; using fc::time_point_sec; @@ -75,7 +73,7 @@ namespace eosio { namespace chain { using fc::flat_map; using fc::flat_multimap; using fc::flat_set; - using fc::static_variant; + using std::variant; using fc::ecc::range_proof_type; using fc::ecc::range_proof_info; using fc::ecc::commitment_type; @@ -84,10 +82,20 @@ namespace eosio { namespace chain { using private_key_type = fc::crypto::private_key; using signature_type = fc::crypto::signature; +#if BOOST_VERSION >= 107100 + // configurable boost deque performs much better than std::deque in our use cases + using block_1024_option_t = boost::container::deque_options< boost::container::block_size<1024u> >::type; + template + using deque = boost::container::deque< T, void, block_1024_option_t >; +#else + template + using deque = std::deque; +#endif + struct void_t{}; using chainbase::allocator; - using shared_string = boost::interprocess::basic_string, allocator>; + using shared_string = chainbase::shared_string; template using shared_vector = boost::interprocess::vector>; template @@ -105,17 +113,10 @@ namespace eosio { namespace chain { shared_blob() = delete; shared_blob(shared_blob&&) = default; - shared_blob(const shared_blob& s) - :shared_string(s.get_allocator()) - { - assign(s.c_str(), s.size()); - } + shared_blob(const shared_blob& s) = default; - shared_blob& operator=(const shared_blob& s) { - assign(s.c_str(), s.size()); - return *this; - } + shared_blob& operator=(const shared_blob& s) = default; shared_blob& operator=(shared_blob&& ) = default; @@ -129,6 +130,71 @@ namespace eosio { namespace chain { {} }; + /* Compares equivalent to any blob that has this as a prefix. */ + struct blob_prefix { + std::string_view _data; + const char* data() const { return _data.data(); } + std::size_t size() const { return _data.size(); } + }; + + /** + * Compare vector, string, string_view, shared_blob (unsigned) + */ + template + inline int compare_blob(const A& a, const B& b) { + static_assert( + std::is_same_v, char> || + std::is_same_v, unsigned char>); + static_assert( + std::is_same_v, char> || + std::is_same_v, unsigned char>); + auto r = memcmp( + a.data(), + b.data(), + std::min(a.size(), b.size())); + if (r) + return r; + if (a.size() < b.size()) + return -1; + if (a.size() > b.size()) + return 1; + return 0; + } + + template + inline int compare_blob(const A& a, const blob_prefix& b) { + static_assert( + std::is_same_v, char> || + std::is_same_v, unsigned char>); + int r = std::memcmp(a.data(), b.data(), std::min(a.size(), b.size())); + if (r) return r; + if (a.size() < b.size()) + return -1; + return 0; + } + + template + inline int compare_blob(const blob_prefix& a, const B& b) { + static_assert( + std::is_same_v, char> || + std::is_same_v, unsigned char>); + int r = std::memcmp(a.data(), b.data(), std::min(a.size(), b.size())); + if (r) return r; + if (a.size() > b.size()) + return 1; + return 0; + } + + /** + * Compare vector, string, string_view, shared_blob (unsigned) + */ + struct unsigned_blob_less { + template + bool operator()(const A& a, const B& b) const { + return compare_blob(a, b) < 0; + } + }; + using action_name = name; using scope_name = name; using account_name = name; @@ -190,6 +256,8 @@ namespace eosio { namespace chain { account_ram_correction_object_type, code_object_type, database_header_object_type, + kv_object_type, + kv_db_config_object_type, OBJECT_TYPE_COUNT ///< Sentry value which contains the number of different object types }; @@ -255,6 +323,7 @@ namespace eosio { namespace chain { using int128_t = __int128; using uint128_t = unsigned __int128; using bytes = vector; + using digests_t = deque; struct sha256_less { bool operator()( const fc::sha256& lhs, const fc::sha256& rhs ) const { @@ -338,7 +407,7 @@ namespace eosio { namespace chain { struct decompose<> { template static auto extract( uint16_t id, const vector& data, ResultVariant& result ) - -> fc::optional + -> std::optional { return {}; } @@ -351,7 +420,7 @@ namespace eosio { namespace chain { template static auto extract( uint16_t id, const vector& data, ResultVariant& result ) - -> fc::optional + -> std::optional { if( id == head_t::extension_id() ) { result = fc::raw::unpack( data ); @@ -395,4 +464,22 @@ namespace eosio { namespace chain { } } // eosio::chain +namespace chainbase { + // chainbase::shared_cow_string + template inline DataStream& operator<<( DataStream& s, const chainbase::shared_cow_string& v ) { + FC_ASSERT( v.size() <= MAX_SIZE_OF_BYTE_ARRAYS ); + fc::raw::pack( s, fc::unsigned_int((uint32_t)v.size())); + if( v.size() ) s.write( v.data(), v.size() ); + return s; + } + + template inline DataStream& operator>>( DataStream& s, chainbase::shared_cow_string& v ) { + fc::unsigned_int size; fc::raw::unpack( s, size ); + FC_ASSERT( size.value <= MAX_SIZE_OF_BYTE_ARRAYS ); + FC_ASSERT( v.size() == 0 ); + v.resize_and_fill(size.value, [&s](char* buf, std::size_t sz) { s.read(buf, sz); }); + return s; + } +} + FC_REFLECT_EMPTY( eosio::chain::void_t ) diff --git a/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp b/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp index f32498dffc4..ee7f0506f35 100644 --- a/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp +++ b/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp @@ -1,11 +1,11 @@ #pragma once #include +#include #include #include #include -#include #include #include @@ -23,13 +23,18 @@ enum class trx_enum_type { unknown = 0, persisted = 1, forked = 2, - aborted = 3 + aborted = 3, + incoming_persisted = 4, + incoming = 5 // incoming_end() needs to be updated if this changes }; +using next_func_t = std::function&)>; + struct unapplied_transaction { const transaction_metadata_ptr trx_meta; const fc::time_point expiry; trx_enum_type trx_type = trx_enum_type::unknown; + next_func_t next; const transaction_id_type& id()const { return trx_meta->id(); } @@ -44,13 +49,6 @@ struct unapplied_transaction { * Persisted are first so that they can be applied in each block until expired. */ class unapplied_transaction_queue { -public: - enum class process_mode { - non_speculative, // HEAD, READ_ONLY, IRREVERSIBLE - speculative_non_producer, // will never produce - speculative_producer // can produce - }; - private: struct by_trx_id; struct by_type; @@ -67,16 +65,13 @@ class unapplied_transaction_queue { > unapplied_trx_queue_type; unapplied_trx_queue_type queue; - process_mode mode = process_mode::speculative_producer; + uint64_t max_transaction_queue_size = 1024*1024*1024; // enforced for incoming + uint64_t size_in_bytes = 0; + size_t incoming_count = 0; public: - void set_mode( process_mode new_mode ) { - if( new_mode != mode ) { - FC_ASSERT( empty(), "set_mode, queue required to be empty" ); - } - mode = new_mode; - } + void set_max_transaction_queue_size( uint64_t v ) { max_transaction_queue_size = v; } bool empty() const { return queue.empty(); @@ -90,14 +85,8 @@ class unapplied_transaction_queue { queue.clear(); } - bool contains_persisted()const { - return queue.get().find( trx_enum_type::persisted ) != queue.get().end(); - } - - bool is_persisted(const transaction_metadata_ptr& trx)const { - auto itr = queue.get().find( trx->id() ); - if( itr == queue.get().end() ) return false; - return itr->trx_type == trx_enum_type::persisted; + size_t incoming_size()const { + return incoming_count; } transaction_metadata_ptr get_trx( const transaction_id_type& id ) const { @@ -109,12 +98,24 @@ class unapplied_transaction_queue { template bool clear_expired( const time_point& pending_block_time, const time_point& deadline, Func&& callback ) { auto& persisted_by_expiry = queue.get(); - while(!persisted_by_expiry.empty() && persisted_by_expiry.begin()->expiry <= pending_block_time) { - if (deadline <= fc::time_point::now()) { + while( !persisted_by_expiry.empty() ) { + const auto& itr = persisted_by_expiry.begin(); + if( itr->expiry > pending_block_time ) { + break; + } + if( deadline <= fc::time_point::now() ) { return false; } - callback( persisted_by_expiry.begin()->id(), persisted_by_expiry.begin()->trx_type ); - persisted_by_expiry.erase( persisted_by_expiry.begin() ); + callback( itr->id(), itr->trx_type ); + if( itr->next ) { + itr->next( std::static_pointer_cast( + std::make_shared( + FC_LOG_MESSAGE( error, "expired transaction ${id}, expiration ${e}, block time ${bt}", + ("id", itr->id())("e", itr->trx_meta->packed_trx()->expiration()) + ("bt", pending_block_time) ) ) ) ); + } + removed( itr ); + persisted_by_expiry.erase( itr ); } return true; } @@ -123,12 +124,22 @@ class unapplied_transaction_queue { if( empty() ) return; auto& idx = queue.get(); for( const auto& receipt : bs->block->transactions ) { - if( receipt.trx.contains() ) { - const auto& pt = receipt.trx.get(); - auto itr = queue.get().find( pt.id() ); - if( itr != queue.get().end() ) { - if( itr->trx_type != trx_enum_type::persisted ) { - idx.erase( pt.id() ); + if( std::holds_alternative(receipt.trx) ) { + const auto& pt = std::get(receipt.trx); + auto itr = idx.find( pt.id() ); + if( itr != idx.end() ) { + if( itr->next ) { + itr->next( std::static_pointer_cast( std::make_shared( + FC_LOG_MESSAGE( info, "duplicate transaction ${id}", ("id", itr->trx_meta->id()))))); + } + if( itr->trx_type != trx_enum_type::persisted && + itr->trx_type != trx_enum_type::incoming_persisted ) { + removed( itr ); + idx.erase( itr ); + } else if( itr->next ) { + idx.modify( itr, [](auto& un){ + un.next = nullptr; + } ); } } } @@ -136,48 +147,106 @@ class unapplied_transaction_queue { } void add_forked( const branch_type& forked_branch ) { - if( mode == process_mode::non_speculative || mode == process_mode::speculative_non_producer ) return; // forked_branch is in reverse order for( auto ritr = forked_branch.rbegin(), rend = forked_branch.rend(); ritr != rend; ++ritr ) { const block_state_ptr& bsptr = *ritr; for( auto itr = bsptr->trxs_metas().begin(), end = bsptr->trxs_metas().end(); itr != end; ++itr ) { const auto& trx = *itr; fc::time_point expiry = trx->packed_trx()->expiration(); - queue.insert( { trx, expiry, trx_enum_type::forked } ); + auto insert_itr = queue.insert( { trx, expiry, trx_enum_type::forked } ); + if( insert_itr.second ) added( insert_itr.first ); } } } - void add_aborted( std::vector aborted_trxs ) { - if( mode == process_mode::non_speculative || mode == process_mode::speculative_non_producer ) return; + void add_aborted( deque aborted_trxs ) { for( auto& trx : aborted_trxs ) { fc::time_point expiry = trx->packed_trx()->expiration(); - queue.insert( { std::move( trx ), expiry, trx_enum_type::aborted } ); + auto insert_itr = queue.insert( { std::move( trx ), expiry, trx_enum_type::aborted } ); + if( insert_itr.second ) added( insert_itr.first ); } } void add_persisted( const transaction_metadata_ptr& trx ) { - if( mode == process_mode::non_speculative ) return; auto itr = queue.get().find( trx->id() ); if( itr == queue.get().end() ) { fc::time_point expiry = trx->packed_trx()->expiration(); - queue.insert( { trx, expiry, trx_enum_type::persisted } ); + auto insert_itr = queue.insert( { trx, expiry, trx_enum_type::persisted } ); + if( insert_itr.second ) added( insert_itr.first ); } else if( itr->trx_type != trx_enum_type::persisted ) { + if (itr->trx_type == trx_enum_type::incoming || itr->trx_type == trx_enum_type::incoming_persisted) + --incoming_count; queue.get().modify( itr, [](auto& un){ un.trx_type = trx_enum_type::persisted; } ); } } + void add_incoming( const transaction_metadata_ptr& trx, bool persist_until_expired, next_func_t next ) { + auto itr = queue.get().find( trx->id() ); + if( itr == queue.get().end() ) { + fc::time_point expiry = trx->packed_trx()->expiration(); + auto insert_itr = queue.insert( + { trx, expiry, persist_until_expired ? trx_enum_type::incoming_persisted : trx_enum_type::incoming, std::move( next ) } ); + if( insert_itr.second ) added( insert_itr.first ); + } else { + if (itr->trx_type != trx_enum_type::incoming && itr->trx_type != trx_enum_type::incoming_persisted) + ++incoming_count; + + queue.get().modify( itr, [persist_until_expired, next{std::move(next)}](auto& un) mutable { + un.trx_type = persist_until_expired ? trx_enum_type::incoming_persisted : trx_enum_type::incoming; + un.next = std::move( next ); + } ); + } + } + using iterator = unapplied_trx_queue_type::index::type::iterator; iterator begin() { return queue.get().begin(); } iterator end() { return queue.get().end(); } + // persisted, forked, aborted + iterator unapplied_begin() { return queue.get().begin(); } + iterator unapplied_end() { return queue.get().upper_bound( trx_enum_type::aborted ); } + iterator persisted_begin() { return queue.get().lower_bound( trx_enum_type::persisted ); } iterator persisted_end() { return queue.get().upper_bound( trx_enum_type::persisted ); } - iterator erase( iterator itr ) { return queue.get().erase( itr ); } + iterator incoming_begin() { return queue.get().lower_bound( trx_enum_type::incoming_persisted ); } + iterator incoming_end() { return queue.get().end(); } // if changed to upper_bound, verify usage performance + + /// caller's responsibilty to call next() if applicable + iterator erase( iterator itr ) { + removed( itr ); + return queue.get().erase( itr ); + } + +private: + template + void added( Itr itr ) { + auto size = calc_size( itr->trx_meta ); + if( itr->trx_type == trx_enum_type::incoming || itr->trx_type == trx_enum_type::incoming_persisted ) { + ++incoming_count; + EOS_ASSERT( size_in_bytes + size < max_transaction_queue_size, tx_resource_exhaustion, + "Transaction ${id}, size ${s} bytes would exceed configured " + "incoming-transaction-queue-size-mb ${qs}, current queue size ${cs} bytes", + ("id", itr->trx_meta->id())("s", size)("qs", max_transaction_queue_size/(1024*1024)) + ("cs", size_in_bytes) ); + } + size_in_bytes += size; + } + + template + void removed( Itr itr ) { + if( itr->trx_type == trx_enum_type::incoming || itr->trx_type == trx_enum_type::incoming_persisted ) { + --incoming_count; + } + size_in_bytes -= calc_size( itr->trx_meta ); + } + + static uint64_t calc_size( const transaction_metadata_ptr& trx ) { + return sizeof(unapplied_transaction) + trx->get_estimated_size(); + } }; diff --git a/libraries/chain/include/eosio/chain/wasm_config.hpp b/libraries/chain/include/eosio/chain/wasm_config.hpp new file mode 100644 index 00000000000..bcfdd9014d3 --- /dev/null +++ b/libraries/chain/include/eosio/chain/wasm_config.hpp @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include + +namespace eosio { namespace chain { + +struct wasm_config { + std::uint32_t max_mutable_global_bytes; + std::uint32_t max_table_elements; + std::uint32_t max_section_elements; + std::uint32_t max_linear_memory_init; + std::uint32_t max_func_local_bytes; + std::uint32_t max_nested_structures; + std::uint32_t max_symbol_bytes; + std::uint32_t max_module_bytes; + std::uint32_t max_code_bytes; + std::uint32_t max_pages; + std::uint32_t max_call_depth; + void validate() const; +}; + +inline constexpr bool operator==(const wasm_config& lhs, const wasm_config& rhs) { + return lhs.max_mutable_global_bytes == rhs.max_mutable_global_bytes && + lhs.max_table_elements == rhs.max_table_elements && + lhs.max_section_elements == rhs.max_section_elements && + lhs.max_linear_memory_init == rhs.max_linear_memory_init && + lhs.max_func_local_bytes == rhs.max_func_local_bytes && + lhs.max_nested_structures == rhs.max_nested_structures && + lhs.max_symbol_bytes == rhs.max_symbol_bytes && + lhs.max_module_bytes == rhs.max_module_bytes && + lhs.max_code_bytes == rhs.max_code_bytes && + lhs.max_pages == rhs.max_pages && + lhs.max_call_depth == rhs.max_call_depth; +} +inline constexpr bool operator!=(const wasm_config& lhs, const wasm_config& rhs) { + return !(lhs == rhs); +} + +}} + +FC_REFLECT(eosio::chain::wasm_config, + (max_mutable_global_bytes) + (max_table_elements) + (max_section_elements) + (max_linear_memory_init) + (max_func_local_bytes) + (max_nested_structures) + (max_symbol_bytes) + (max_module_bytes) + (max_code_bytes) + (max_pages) + (max_call_depth) +) diff --git a/libraries/chain/include/eosio/chain/wasm_eosio_binary_ops.hpp b/libraries/chain/include/eosio/chain/wasm_eosio_binary_ops.hpp index cc6149f5ba9..36c6327981c 100644 --- a/libraries/chain/include/eosio/chain/wasm_eosio_binary_ops.hpp +++ b/libraries/chain/include/eosio/chain/wasm_eosio_binary_ops.hpp @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include diff --git a/libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp b/libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp index 71ab4ab677f..f42ba5aaba6 100644 --- a/libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp +++ b/libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp @@ -27,8 +27,8 @@ namespace eosio { namespace chain { namespace wasm_injections { static std::map injected_index_mapping; static uint32_t next_injected_index; - static void init( Module& mod ) { - type_slots.clear(); + static void init( Module& mod ) { + type_slots.clear(); registered_injected.clear(); injected_index_mapping.clear(); build_type_slots( mod ); @@ -42,7 +42,7 @@ namespace eosio { namespace chain { namespace wasm_injections { for ( auto param : mod.types[i]->parameters ) type_slot_list.push_back( static_cast(param) ); type_slots.emplace( type_slot_list, i ); - } + } } template @@ -67,10 +67,10 @@ namespace eosio { namespace chain { namespace wasm_injections { int actual_index; get_next_indices( module, index, actual_index ); registered_injected.emplace( func_name, index ); - decltype(module.functions.imports) new_import = { {{func_type_index}, EOSIO_INJECTED_MODULE_NAME, std::move(func_name)} }; + decltype(module.functions.imports) new_import = { {{func_type_index}, eosio_injected_module_name, std::move(func_name)} }; // prepend to the head of the imports - module.functions.imports.insert( module.functions.imports.begin()+(registered_injected.size()-1), new_import.begin(), new_import.end() ); - injected_index_mapping.emplace( index, actual_index ); + module.functions.imports.insert( module.functions.imports.begin()+(registered_injected.size()-1), new_import.begin(), new_import.end() ); + injected_index_mapping.emplace( index, actual_index ); // shift all exported functions by 1 for ( size_t i=0; i < module.exports.size(); i++ ) { @@ -95,7 +95,7 @@ namespace eosio { namespace chain { namespace wasm_injections { } } }; - + struct noop_injection_visitor { static void inject( IR::Module& m ); static void initializer(); @@ -110,7 +110,7 @@ namespace eosio { namespace chain { namespace wasm_injections { static void inject( IR::Module& m ); static void initializer(); }; - + struct tables_injection_visitor { static void inject( IR::Module& m ); static void initializer(); @@ -121,11 +121,6 @@ namespace eosio { namespace chain { namespace wasm_injections { static void initializer(); }; - struct max_memory_injection_visitor { - static void inject( IR::Module& m ); - static void initializer(); - }; - struct blacklist_injection_visitor { static void inject( IR::Module& m ); static void initializer(); @@ -133,8 +128,8 @@ namespace eosio { namespace chain { namespace wasm_injections { using wasm_validate_func = std::function; - - // just pass + + // just pass struct no_injections_injectors { static void inject( IR::Module& m ) {} }; @@ -154,10 +149,10 @@ namespace eosio { namespace chain { namespace wasm_injections { struct instruction_counter { static constexpr bool kills = false; static constexpr bool post = false; - static void init() { - icnt=0; - tcnt=0; - bcnt=0; + static void init() { + icnt=0; + tcnt=0; + bcnt=0; while ( !fcnts.empty() ) fcnts.pop(); } @@ -168,7 +163,7 @@ namespace eosio { namespace chain { namespace wasm_injections { static uint32_t icnt; /* instructions so far */ static uint32_t tcnt; /* total instructions */ static uint32_t bcnt; /* total instructions from block types */ - static std::queue fcnts; + static std::queue fcnts; }; struct checktime_block_type { @@ -178,7 +173,7 @@ namespace eosio { namespace chain { namespace wasm_injections { while( !x.empty() ) \ x.pop() - static void init() { + static void init() { CLEAR(block_stack); CLEAR(type_stack); CLEAR(orderings); @@ -204,7 +199,7 @@ namespace eosio { namespace chain { namespace wasm_injections { static constexpr bool post = false; static void init() {} static void accept( wasm_ops::instr* inst, wasm_ops::visitor_arg& arg ) { - if ( checktime_block_type::type_stack.empty() ) + if ( checktime_block_type::type_stack.empty() ) return; if ( !checktime_block_type::type_stack.top() ) { // empty or is not a loop checktime_block_type::block_stack.pop(); @@ -240,7 +235,7 @@ namespace eosio { namespace chain { namespace wasm_injections { static void accept( wasm_ops::instr* inst, wasm_ops::visitor_arg& arg ) { auto mapped_index = injector_utils::injected_index_mapping.find(chktm_idx); - wasm_ops::op_types<>::call_t chktm; + wasm_ops::op_types<>::call_t chktm; chktm.field = mapped_index->second; chktm.pack(arg.new_code); } @@ -266,35 +261,30 @@ namespace eosio { namespace chain { namespace wasm_injections { } }; - + struct call_depth_check_and_insert_checktime { static constexpr bool kills = true; static constexpr bool post = false; - static int32_t global_idx; - static void init() { - global_idx = -1; + static int32_t global_idx; + static void init( Module& mod) { + mod.globals.defs.push_back({{ValueType::i32, true}, {(I32)eosio::chain::wasm_constraints::maximum_call_depth}}); + global_idx = mod.globals.size()-1; } static void accept( wasm_ops::instr* inst, wasm_ops::visitor_arg& arg ) { - if ( global_idx == -1 ) { - arg.module->globals.defs.push_back({{ValueType::i32, true}, {(I32) eosio::chain::wasm_constraints::maximum_call_depth}}); - } - - global_idx = arg.module->globals.size()-1; - int32_t assert_idx; injector_utils::add_import(*(arg.module), "call_depth_assert", assert_idx); wasm_ops::op_types<>::call_t call_assert; wasm_ops::op_types<>::call_t call_checktime; - wasm_ops::op_types<>::get_global_t get_global_inst; + wasm_ops::op_types<>::get_global_t get_global_inst; wasm_ops::op_types<>::set_global_t set_global_inst; - wasm_ops::op_types<>::i32_eqz_t eqz_inst; - wasm_ops::op_types<>::i32_const_t const_inst; + wasm_ops::op_types<>::i32_eqz_t eqz_inst; + wasm_ops::op_types<>::i32_const_t const_inst; wasm_ops::op_types<>::i32_add_t add_inst; wasm_ops::op_types<>::end_t end_inst; - wasm_ops::op_types<>::if__t if_inst; - wasm_ops::op_types<>::else__t else_inst; + wasm_ops::op_types<>::if__t if_inst; + wasm_ops::op_types<>::else__t else_inst; call_assert.field = assert_idx; call_checktime.field = checktime_injection::chktm_idx; @@ -327,7 +317,7 @@ namespace eosio { namespace chain { namespace wasm_injections { } const_inst.field = 1; - INSERT_INJECTED(get_global_inst); + INSERT_INJECTED(get_global_inst); INSERT_INJECTED(const_inst); INSERT_INJECTED(add_inst); INSERT_INJECTED(set_global_inst); @@ -335,7 +325,7 @@ namespace eosio { namespace chain { namespace wasm_injections { #undef INSERT_INJECTED } - }; + }; // float injections constexpr const char* inject_which_op( uint16_t opcode ) { @@ -662,7 +652,7 @@ namespace eosio { namespace chain { namespace wasm_injections { f32promote.pack(arg.new_code); } }; - + struct f64_demote_injector { static constexpr bool kills = true; static constexpr bool post = false; @@ -677,7 +667,7 @@ namespace eosio { namespace chain { namespace wasm_injections { }; struct pre_op_injectors : wasm_ops::op_types { - // float binops + // float binops using f32_add_t = wasm_ops::f32_add >; using f32_sub_t = wasm_ops::f32_sub >; using f32_div_t = wasm_ops::f32_div >; @@ -701,7 +691,7 @@ namespace eosio { namespace chain { namespace wasm_injections { using f32_gt_t = wasm_ops::f32_gt >; using f32_ge_t = wasm_ops::f32_ge >; - // float binops + // float binops using f64_add_t = wasm_ops::f64_add >; using f64_sub_t = wasm_ops::f64_sub >; using f64_div_t = wasm_ops::f64_div >; @@ -729,7 +719,7 @@ namespace eosio { namespace chain { namespace wasm_injections { using f64_promote_f32_t = wasm_ops::f64_promote_f32 ; using f32_demote_f64_t = wasm_ops::f32_demote_f64 ; - + using i32_trunc_s_f32_t = wasm_ops::i32_trunc_s_f32 >; using i32_trunc_u_f32_t = wasm_ops::i32_trunc_u_f32 >; using i32_trunc_s_f64_t = wasm_ops::i32_trunc_s_f64 >; @@ -738,7 +728,7 @@ namespace eosio { namespace chain { namespace wasm_injections { using i64_trunc_u_f32_t = wasm_ops::i64_trunc_u_f32 >; using i64_trunc_s_f64_t = wasm_ops::i64_trunc_s_f64 >; using i64_trunc_u_f64_t = wasm_ops::i64_trunc_u_f64 >; - + using f32_convert_s_i32 = wasm_ops::f32_convert_s_i32 >; using f32_convert_s_i64 = wasm_ops::f32_convert_s_i64 >; using f32_convert_u_i32 = wasm_ops::f32_convert_u_i32 >; @@ -777,24 +767,23 @@ namespace eosio { namespace chain { namespace wasm_injections { } } }; - + // "full injection" gives checktime & depth counting along with softfloat injection. // Otherwise you'll just get softfloat injection template class wasm_binary_injection { - using standard_module_injectors = module_injectors< max_memory_injection_visitor >; public: - wasm_binary_injection( IR::Module& mod ) : _module( &mod ) { - _module_injectors.init(); + wasm_binary_injection( IR::Module& mod ) : _module( &mod ) { // initialize static fields of injectors injector_utils::init( mod ); checktime_injection::init(); - call_depth_check_and_insert_checktime::init(); + if constexpr (full_injection) { + call_depth_check_and_insert_checktime::init( mod ); + } } void inject() { - _module_injectors.inject( *_module ); // inject checktime first if constexpr (full_injection) injector_utils::add_import( *_module, u8"checktime", checktime_injection::chktm_idx ); @@ -844,7 +833,6 @@ namespace eosio { namespace chain { namespace wasm_injections { } private: IR::Module* _module; - standard_module_injectors _module_injectors; }; }}} // namespace wasm_constraints, chain, eosio diff --git a/libraries/chain/include/eosio/chain/wasm_interface.hpp b/libraries/chain/include/eosio/chain/wasm_interface.hpp index e0bbc67f87a..ed01f3bd1dd 100644 --- a/libraries/chain/include/eosio/chain/wasm_interface.hpp +++ b/libraries/chain/include/eosio/chain/wasm_interface.hpp @@ -3,9 +3,6 @@ #include #include #include -#if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED) -#include -#endif #include "Runtime/Linker.h" #include "Runtime/Runtime.h" @@ -20,55 +17,6 @@ namespace eosio { namespace chain { int32_t code = 0; }; - namespace webassembly { namespace common { - class intrinsics_accessor; - - class root_resolver : public Runtime::Resolver { - public: - // The non-default constructor puts root_resolver in a mode where it does validation, i.e. only allows "env" imports. - // This mode is used by the generic validating code that runs during setcode, where we only want "env" to pass. - // The default constructor is used when no validation is required such as when the wavm runtime needs to - // allow linkage to the intrinsics and the injected functions. - - root_resolver() {} - - root_resolver( const whitelisted_intrinsics_type& whitelisted_intrinsics ) - :whitelisted_intrinsics(&whitelisted_intrinsics) - {} - - bool resolve(const string& mod_name, - const string& export_name, - IR::ObjectType type, - Runtime::ObjectInstance*& out) override - { try { - bool fail = false; - - if( whitelisted_intrinsics != nullptr ) { - // Protect access to "private" injected functions; so for now just simply allow "env" since injected - // functions are in a different module. - EOS_ASSERT( mod_name == "env", wasm_exception, - "importing from module that is not 'env': ${module}.${export}", - ("module",mod_name)("export",export_name) ); - - // Only consider imports that are in the whitelisted set of intrinsics - fail = !is_intrinsic_whitelisted( *whitelisted_intrinsics, export_name ); - } - - // Try to resolve an intrinsic first. - if( !fail && Runtime::IntrinsicResolver::singleton.resolve( mod_name, export_name, type, out ) ) { - return true; - } - - EOS_THROW( wasm_exception, "${module}.${export} unresolveable", - ("module",mod_name)("export",export_name) ); - return false; - } FC_CAPTURE_AND_RETHROW( (mod_name)(export_name) ) } - - protected: - const whitelisted_intrinsics_type* whitelisted_intrinsics = nullptr; - }; - } } - /** * @class wasm_interface * @@ -76,7 +24,6 @@ namespace eosio { namespace chain { class wasm_interface { public: enum class vm_type { - wabt, eos_vm, eos_vm_jit, eos_vm_oc @@ -87,12 +34,10 @@ namespace eosio { namespace chain { switch (vmtype) { case vm_type::eos_vm: return "eos-vm"; - case vm_type::eos_vm_jit: - return "eos-vm-jit"; case vm_type::eos_vm_oc: return "eos-vm-oc"; default: - return "wabt"; + return "eos-vm-jit"; } } @@ -119,7 +64,6 @@ namespace eosio { namespace chain { private: unique_ptr my; - friend class eosio::chain::webassembly::common::intrinsics_accessor; }; } } // eosio::chain @@ -128,4 +72,4 @@ namespace eosio{ namespace chain { std::istream& operator>>(std::istream& in, wasm_interface::vm_type& runtime); }} -FC_REFLECT_ENUM( eosio::chain::wasm_interface::vm_type, (wabt)(eos_vm)(eos_vm_jit)(eos_vm_oc) ) +FC_REFLECT_ENUM( eosio::chain::wasm_interface::vm_type, (eos_vm)(eos_vm_jit)(eos_vm_oc) ) diff --git a/libraries/chain/include/eosio/chain/wasm_interface_private.hpp b/libraries/chain/include/eosio/chain/wasm_interface_private.hpp index 793a8a28a86..3e15ed824de 100644 --- a/libraries/chain/include/eosio/chain/wasm_interface_private.hpp +++ b/libraries/chain/include/eosio/chain/wasm_interface_private.hpp @@ -1,8 +1,6 @@ #pragma once #include -#include -#include #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED #include #else @@ -12,6 +10,7 @@ #include #include #include +#include #include #include @@ -21,12 +20,8 @@ #include "WAST/WAST.h" #include "IR/Validate.h" -#if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED) #include #include -#else -#define _REGISTER_EOS_VM_INTRINSIC(CLS, MOD, METHOD, WASM_SIG, NAME, SIG) -#endif using namespace fc; using namespace eosio::chain::webassembly; @@ -54,7 +49,9 @@ namespace eosio { namespace chain { #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED struct eosvmoc_tier { - eosvmoc_tier(const boost::filesystem::path& d, const eosvmoc::config& c, const chainbase::database& db) : cc(d, c, db), exec(cc) {} + eosvmoc_tier(const boost::filesystem::path& d, const eosvmoc::config& c, const chainbase::database& db) + : cc(d, c, db), exec(cc), + mem(wasm_constraints::maximum_linear_memory/wasm_constraints::wasm_page_size) {} eosvmoc::code_cache_async cc; eosvmoc::executor exec; eosvmoc::memory mem; @@ -62,8 +59,6 @@ namespace eosio { namespace chain { #endif wasm_interface_impl(wasm_interface::vm_type vm, bool eosvmoc_tierup, const chainbase::database& d, const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config) : db(d), wasm_runtime_time(vm) { - if(vm == wasm_interface::vm_type::wabt) - runtime_interface = std::make_unique(); #ifdef EOSIO_EOS_VM_RUNTIME_ENABLED if(vm == wasm_interface::vm_type::eos_vm) runtime_interface = std::make_unique>(); @@ -166,6 +161,7 @@ namespace eosio { namespace chain { try { Serialization::MemoryInputStream stream((const U8*)bytes.data(), bytes.size()); + WASM::scoped_skip_checks no_check; WASM::serialize(stream, module); module.userSections.clear(); } catch (const Serialization::FatalSerializationException& e) { @@ -217,52 +213,8 @@ namespace eosio { namespace chain { const wasm_interface::vm_type wasm_runtime_time; #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED - fc::optional eosvmoc; + std::optional eosvmoc; #endif }; -#define _ADD_PAREN_1(...) ((__VA_ARGS__)) _ADD_PAREN_2 -#define _ADD_PAREN_2(...) ((__VA_ARGS__)) _ADD_PAREN_1 -#define _ADD_PAREN_1_END -#define _ADD_PAREN_2_END -#define _WRAPPED_SEQ(SEQ) BOOST_PP_CAT(_ADD_PAREN_1 SEQ, _END) - -#define _REGISTER_INTRINSIC_EXPLICIT(CLS, MOD, METHOD, WASM_SIG, NAME, SIG)\ - _REGISTER_WAVM_INTRINSIC(CLS, MOD, METHOD, WASM_SIG, NAME, SIG) \ - _REGISTER_WABT_INTRINSIC(CLS, MOD, METHOD, WASM_SIG, NAME, SIG) \ - _REGISTER_EOS_VM_INTRINSIC(CLS, MOD, METHOD, WASM_SIG, NAME, SIG) \ - _REGISTER_EOSVMOC_INTRINSIC(CLS, MOD, METHOD, WASM_SIG, NAME, SIG) - -#define _REGISTER_INTRINSIC4(CLS, MOD, METHOD, WASM_SIG, NAME, SIG)\ - _REGISTER_INTRINSIC_EXPLICIT(CLS, MOD, METHOD, WASM_SIG, NAME, SIG ) - -#define _REGISTER_INTRINSIC3(CLS, MOD, METHOD, WASM_SIG, NAME)\ - _REGISTER_INTRINSIC_EXPLICIT(CLS, MOD, METHOD, WASM_SIG, NAME, decltype(&CLS::METHOD) ) - -#define _REGISTER_INTRINSIC2(CLS, MOD, METHOD, WASM_SIG)\ - _REGISTER_INTRINSIC_EXPLICIT(CLS, MOD, METHOD, WASM_SIG, BOOST_PP_STRINGIZE(METHOD), decltype(&CLS::METHOD) ) - -#define _REGISTER_INTRINSIC1(CLS, MOD, METHOD)\ - static_assert(false, "Cannot register " BOOST_PP_STRINGIZE(CLS) ":" BOOST_PP_STRINGIZE(METHOD) " without a signature"); - -#define _REGISTER_INTRINSIC0(CLS, MOD, METHOD)\ - static_assert(false, "Cannot register " BOOST_PP_STRINGIZE(CLS) ": without a method name and signature"); - -#define _UNWRAP_SEQ(...) __VA_ARGS__ - -#define _EXPAND_ARGS(CLS, MOD, INFO)\ - ( CLS, MOD, _UNWRAP_SEQ INFO ) - -#define _REGISTER_INTRINSIC(R, CLS, INFO)\ - BOOST_PP_CAT(BOOST_PP_OVERLOAD(_REGISTER_INTRINSIC, _UNWRAP_SEQ INFO) _EXPAND_ARGS(CLS, "env", INFO), BOOST_PP_EMPTY()) - -#define REGISTER_INTRINSICS(CLS, MEMBERS)\ - BOOST_PP_SEQ_FOR_EACH(_REGISTER_INTRINSIC, CLS, _WRAPPED_SEQ(MEMBERS)) - -#define _REGISTER_INJECTED_INTRINSIC(R, CLS, INFO)\ - BOOST_PP_CAT(BOOST_PP_OVERLOAD(_REGISTER_INTRINSIC, _UNWRAP_SEQ INFO) _EXPAND_ARGS(CLS, EOSIO_INJECTED_MODULE_NAME, INFO), BOOST_PP_EMPTY()) - -#define REGISTER_INJECTED_INTRINSICS(CLS, MEMBERS)\ - BOOST_PP_SEQ_FOR_EACH(_REGISTER_INJECTED_INTRINSIC, CLS, _WRAPPED_SEQ(MEMBERS)) - } } // eosio::chain diff --git a/libraries/chain/include/eosio/chain/webassembly/common.hpp b/libraries/chain/include/eosio/chain/webassembly/common.hpp index 7f441be8649..699ba8a598c 100644 --- a/libraries/chain/include/eosio/chain/webassembly/common.hpp +++ b/libraries/chain/include/eosio/chain/webassembly/common.hpp @@ -2,98 +2,123 @@ #include #include - -#define EOSIO_INJECTED_MODULE_NAME "eosio_injection" +#include +#include +#include +#include using namespace fc; +namespace eosio { namespace chain { namespace webassembly { + // forward declaration + class interface; +}}} // ns eosio::chain::webassembly + +// TODO need to fix up wasm_tests to not use this macro +#define EOSIO_INJECTED_MODULE_NAME "eosio_injection" + namespace eosio { namespace chain { class apply_context; class transaction_context; - template - struct class_from_wasm { - /** - * by default this is just constructing an object - * @param wasm - the wasm_interface to use - * @return - */ - static auto value(apply_context& ctx) { - return T(ctx); - } + inline static constexpr auto eosio_injected_module_name = EOSIO_INJECTED_MODULE_NAME; + + template + using span = eosio::vm::span; + + template + using legacy_ptr = eosio::vm::argument_proxy; + + template + using legacy_span = eosio::vm::argument_proxy, Align>; + + struct null_terminated_ptr { + null_terminated_ptr(const char* ptr) : ptr(ptr) {} + const char* data() const { return ptr; } + const char* ptr; }; - template<> - struct class_from_wasm { - /** - * by default this is just constructing an object - * @param wasm - the wasm_interface to use - * @return - */ - template - static auto &value(ApplyCtx& ctx) { - return ctx.trx_context; - } + struct memcpy_params { + void* dst; + const void* src; + vm::wasm_size_t size; }; - template<> - struct class_from_wasm { - /** - * Don't construct a new apply_context, just return a reference to the existing ont - * @param wasm - * @return - */ - static auto &value(apply_context& ctx) { - return ctx; - } + struct memcmp_params { + const void* lhs; + const void* rhs; + vm::wasm_size_t size; }; - /** - * class to represent an in-wasm-memory array - * it is a hint to the transcriber that the next parameter will - * be a size (data bytes length) and that the pair are validated together - * This triggers the template specialization of intrinsic_invoker_impl - * @tparam T - */ - template - struct array_ptr { - explicit array_ptr (T * value) : value(value) {} - - typename std::add_lvalue_reference::type operator*() const { - return *value; - } + struct memset_params { + const void* dst; + const int32_t val; + vm::wasm_size_t size; + }; - T *operator->() const noexcept { - return value; + // define the type converter for eosio + struct type_converter : public eosio::vm::type_converter { + using base_type = eosio::vm::type_converter; + using base_type::type_converter; + using base_type::from_wasm; + using base_type::to_wasm; + using base_type::as_value; + using base_type::as_result; + using base_type::elem_type; + using base_type::get_host; + + EOS_VM_FROM_WASM(memcpy_params, (void* dst, const void* src, vm::wasm_size_t size)) { + validate_pointer(dst, size); + validate_pointer(src, size); + validate_pointer(dst, 1); + return { dst, src, size }; } - operator T *() const { - return value; + EOS_VM_FROM_WASM(memcmp_params, (const void* lhs, const void* rhs, vm::wasm_size_t size)) { + validate_pointer(lhs, size); + validate_pointer(rhs, size); + return { lhs, rhs, size }; } - T *value; - }; - - /** - * class to represent an in-wasm-memory char array that must be null terminated - */ - struct null_terminated_ptr { - explicit null_terminated_ptr(char* value) : value(value) {} - - typename std::add_lvalue_reference::type operator*() const { - return *value; + EOS_VM_FROM_WASM(memset_params, (void* dst, int32_t val, vm::wasm_size_t size)) { + validate_pointer(dst, size); + validate_pointer(dst, 1); + return { dst, val, size }; } - char *operator->() const noexcept { - return value; + template + auto from_wasm(void* ptr) const + -> std::enable_if_t< std::is_pointer_v, + vm::argument_proxy > { + validate_pointer>(ptr, 1); + return {ptr}; } - operator char *() const { - return value; + template + auto from_wasm(void* ptr, vm::tag = {}) const + -> std::enable_if_t< vm::is_argument_proxy_type_v && + std::is_pointer_v, T> { + if constexpr(T::is_legacy()) { + EOS_ASSERT(ptr != this->get_interface().get_memory(), wasm_execution_error, "references cannot be created for null pointers"); + } + this->template validate_pointer(ptr, 1); + return {ptr}; } - char *value; + EOS_VM_FROM_WASM(null_terminated_ptr, (const void* ptr)) { + validate_null_terminated_pointer(ptr); + return {static_cast(ptr)}; + } + EOS_VM_FROM_WASM(name, (uint64_t e)) { return name{e}; } + uint64_t to_wasm(name&& n) { return n.to_uint64_t(); } + EOS_VM_FROM_WASM(float32_t, (float f)) { return ::to_softfloat32(f); } + EOS_VM_FROM_WASM(float64_t, (double f)) { return ::to_softfloat64(f); } }; - } } // eosio::chain + using eos_vm_host_functions_t = eosio::vm::registered_host_functions; + using wasm_size_t = eosio::vm::wasm_size_t; + +}} // eosio::chain diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc.hpp index 6ecb9b8f60d..10e73d1b1ee 100644 --- a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc.hpp +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc.hpp @@ -14,14 +14,13 @@ #include #include -#include +#include namespace eosio { namespace chain { namespace webassembly { namespace eosvmoc { using namespace IR; using namespace Runtime; using namespace fc; -using namespace eosio::chain::webassembly::common; using namespace eosio::chain::eosvmoc; @@ -44,14 +43,11 @@ class eosvmoc_runtime : public eosio::chain::wasm_runtime_interface { }; /** - * class to represent an in-wasm-memory array - * it is a hint to the transcriber that the next parameter will - * be a size (in Ts) and that the pair are validated together - * This triggers the template specialization of intrinsic_invoker_impl + * validate an in-wasm-memory array * @tparam T */ template -inline array_ptr array_ptr_impl (size_t ptr, size_t length) +inline void* array_ptr_impl (size_t ptr, size_t length) { constexpr int cb_full_linear_memory_start_segment_offset = OFFSET_OF_CONTROL_BLOCK_MEMBER(full_linear_memory_start); constexpr int cb_first_invalid_memory_address_segment_offset = OFFSET_OF_CONTROL_BLOCK_MEMBER(first_invalid_memory_address); @@ -60,25 +56,24 @@ inline array_ptr array_ptr_impl (size_t ptr, size_t length) asm volatile("cmp %%gs:%c[firstInvalidMemory], %[End]\n" "jle 1f\n" - "mov %%gs:%c[maximumEosioWasmMemory], %[Ptr]\n" //always invalid address + "mov %%gs:(%[End]), %[Ptr]\n" // invalid pointer if out of range "1:\n" "add %%gs:%c[linearMemoryStart], %[Ptr]\n" : [Ptr] "+r" (ptr), [End] "+r" (end) : [linearMemoryStart] "i" (cb_full_linear_memory_start_segment_offset), - [firstInvalidMemory] "i" (cb_first_invalid_memory_address_segment_offset), - [maximumEosioWasmMemory] "i" (wasm_constraints::maximum_linear_memory) + [firstInvalidMemory] "i" (cb_first_invalid_memory_address_segment_offset) : "cc" ); - return array_ptr((T*)ptr); + return (void*)ptr; } /** - * class to represent an in-wasm-memory char array that must be null terminated + * validate an in-wasm-memory char array that must be null terminated */ -inline null_terminated_ptr null_terminated_ptr_impl(uint64_t ptr) +inline char* null_terminated_ptr_impl(uint64_t ptr) { constexpr int cb_full_linear_memory_start_segment_offset = OFFSET_OF_CONTROL_BLOCK_MEMBER(full_linear_memory_start); constexpr int cb_first_invalid_memory_address_segment_offset = OFFSET_OF_CONTROL_BLOCK_MEMBER(first_invalid_memory_address); @@ -105,103 +100,7 @@ inline null_terminated_ptr null_terminated_ptr_impl(uint64_t ptr) : "cc" ); - return null_terminated_ptr((char*)ptr); -} - -template -struct void_ret_wrapper { - using type = T; -}; - -template<> -struct void_ret_wrapper { - using type = char; -}; - -template -using void_ret_wrapper_t = typename void_ret_wrapper::type; - -/** - * template that maps native types to WASM VM types - * @tparam T the native type - */ -template -struct native_to_wasm { - using type = void; -}; - -/** - * specialization for mapping pointers to int32's - */ -template -struct native_to_wasm { - using type = I32; -}; - -/** - * Mappings for native types - */ -template<> -struct native_to_wasm { - using type = F32; -}; -template<> -struct native_to_wasm { - using type = F64; -}; -template<> -struct native_to_wasm { - using type = I32; -}; -template<> -struct native_to_wasm { - using type = I32; -}; -template<> -struct native_to_wasm { - using type = I64; -}; -template<> -struct native_to_wasm { - using type = I64; -}; -template<> -struct native_to_wasm { - using type = I32; -}; -template<> -struct native_to_wasm { - using type = I64; -}; -template<> -struct native_to_wasm { - using type = I64; -}; -template<> -struct native_to_wasm { - using type = I32; -}; - -template<> -struct native_to_wasm { - using type = I32; -}; - -// convenience alias -template -using native_to_wasm_t = typename native_to_wasm::type; - -template -inline auto convert_native_to_wasm(T val) { - return native_to_wasm_t(val); -} - -inline auto convert_native_to_wasm(const name &val) { - return native_to_wasm_t(val.to_uint64_t()); -} - -inline auto convert_native_to_wasm(const fc::time_point_sec& val) { - return native_to_wasm_t(val.sec_since_epoch()); + return (char*)ptr; } inline auto convert_native_to_wasm(char* ptr) { @@ -212,14 +111,9 @@ inline auto convert_native_to_wasm(char* ptr) { : [fullLinearMemOffset] "i" (cb_full_linear_memory_start_offset) ); U64 delta = (U64)(ptr - full_linear_memory_start); - array_ptr_impl(delta, 1); return (U32)delta; } -template -inline auto convert_wasm_to_native(native_to_wasm_t val) { - return T(val); -} template struct wasm_to_value_type; @@ -234,11 +128,11 @@ struct wasm_to_value_type { static constexpr auto value = ValueType::f64; }; template<> -struct wasm_to_value_type { +struct wasm_to_value_type { static constexpr auto value = ValueType::i32; }; template<> -struct wasm_to_value_type { +struct wasm_to_value_type { static constexpr auto value = ValueType::i64; }; @@ -256,62 +150,22 @@ struct wasm_to_rvalue_type { static constexpr auto value = ResultType::f64; }; template<> -struct wasm_to_rvalue_type { +struct wasm_to_rvalue_type { static constexpr auto value = ResultType::i32; }; template<> -struct wasm_to_rvalue_type { +struct wasm_to_rvalue_type { static constexpr auto value = ResultType::i64; }; template<> struct wasm_to_rvalue_type { static constexpr auto value = ResultType::none; }; -template<> -struct wasm_to_rvalue_type { - static constexpr auto value = ResultType::i64; -}; -template<> -struct wasm_to_rvalue_type { - static constexpr auto value = ResultType::i64; -}; - -template<> -struct wasm_to_rvalue_type { - static constexpr auto value = ResultType::i32; -}; - -template<> -struct wasm_to_rvalue_type { - static constexpr auto value = ResultType::i32; -}; template constexpr auto wasm_to_rvalue_type_v = wasm_to_rvalue_type::value; -template -struct is_reference_from_value { - static constexpr bool value = false; -}; - -template<> -struct is_reference_from_value { - static constexpr bool value = true; -}; - -template<> -struct is_reference_from_value { - static constexpr bool value = true; -}; - -template -constexpr bool is_reference_from_value_v = is_reference_from_value::value; - - - -struct void_type { -}; /** * Forward declaration of provider for FunctionType given a desired C ABI signature @@ -329,471 +183,198 @@ struct wasm_function_type_provider { } }; -/** - * Forward declaration of the invoker type which transcribes arguments to/from a native method - * and injects the appropriate checks - * - * @tparam Ret - the return type of the native function - * @tparam NativeParameters - a std::tuple of the remaining native parameters to transcribe - * @tparam WasmParameters - a std::tuple of the transribed parameters - */ -template -struct intrinsic_invoker_impl; - -/** - * Specialization for the fully transcribed signature - * @tparam Ret - the return type of the native function - * @tparam Translated - the arguments to the wasm function - */ -template -struct intrinsic_invoker_impl, std::tuple> { - using next_method_type = Ret (*)(Translated...); - - template - static native_to_wasm_t invoke(Translated... translated) { - try { - if constexpr(!is_injected) { - constexpr int cb_current_call_depth_remaining_segment_offset = OFFSET_OF_CONTROL_BLOCK_MEMBER(current_call_depth_remaining); - constexpr int depth_assertion_intrinsic_offset = OFFSET_OF_FIRST_INTRINSIC - (int)boost::hana::index_if(intrinsic_table, ::boost::hana::equal.to(BOOST_HANA_STRING("eosvmoc_internal.depth_assert"))).value()*8; - asm volatile("cmpl $1,%%gs:%c[callDepthRemainOffset]\n" - "jne 1f\n" - "callq *%%gs:%c[depthAssertionIntrinsicOffset]\n" - "1:\n" - : - : [callDepthRemainOffset] "i" (cb_current_call_depth_remaining_segment_offset), - [depthAssertionIntrinsicOffset] "i" (depth_assertion_intrinsic_offset) - : "cc"); - } - return convert_native_to_wasm(Method(translated...)); - } - catch(...) { - *reinterpret_cast(eos_vm_oc_get_exception_ptr()) = std::current_exception(); - } - siglongjmp(*eos_vm_oc_get_jmp_buf(), EOSVMOC_EXIT_EXCEPTION); - __builtin_unreachable(); - } - - template - static const auto fn() { - return invoke; - } +struct eos_vm_oc_execution_interface { + inline const auto& operand_from_back(std::size_t index) const { return *(os - index - 1); } + eosio::vm::native_value* os; }; -/** - * specialization of the fully transcribed signature for void return values - * @tparam Translated - the arguments to the wasm function - */ -template -struct intrinsic_invoker_impl, std::tuple> { - using next_method_type = void_type (*)(Translated...); - - template - static void invoke(Translated... translated) { - try { - if constexpr(!is_injected) { - constexpr int cb_current_call_depth_remaining_segment_offset = OFFSET_OF_CONTROL_BLOCK_MEMBER(current_call_depth_remaining); - constexpr int depth_assertion_intrinsic_offset = OFFSET_OF_FIRST_INTRINSIC - (int)boost::hana::index_if(intrinsic_table, ::boost::hana::equal.to(BOOST_HANA_STRING("eosvmoc_internal.depth_assert"))).value()*8; - asm volatile("cmpl $1,%%gs:%c[callDepthRemainOffset]\n" - "jne 1f\n" - "callq *%%gs:%c[depthAssertionIntrinsicOffset]\n" - "1:\n" - : - : [callDepthRemainOffset] "i" (cb_current_call_depth_remaining_segment_offset), - [depthAssertionIntrinsicOffset] "i" (depth_assertion_intrinsic_offset) - : "cc"); - } - Method(translated...); - return; - } - catch(...) { - *reinterpret_cast(eos_vm_oc_get_exception_ptr()) = std::current_exception(); - } - siglongjmp(*eos_vm_oc_get_jmp_buf(), EOSVMOC_EXIT_EXCEPTION); - __builtin_unreachable(); - } +struct eos_vm_oc_type_converter : public eosio::vm::type_converter { + using base_type = eosio::vm::type_converter; + using base_type::type_converter; + using base_type::to_wasm; + using base_type::as_result; + using base_type::get_host; - template - static const auto fn() { - return invoke; - } -}; + EOS_VM_FROM_WASM(bool, (uint32_t value)) { return value ? 1 : 0; } -/** - * Sepcialization for transcribing a simple type in the native method signature - * @tparam Ret - the return type of the native method - * @tparam Input - the type of the native parameter to transcribe - * @tparam Inputs - the remaining native parameters to transcribe - * @tparam Translated - the list of transcribed wasm parameters - */ -template -struct intrinsic_invoker_impl, std::tuple> { - using translated_type = native_to_wasm_t; - using next_step = intrinsic_invoker_impl, std::tuple>; - using then_type = Ret (*)(Input, Inputs..., Translated...); - - template - static Ret translate_one(Inputs... rest, Translated... translated, translated_type last) { - auto native = convert_wasm_to_native(last); - return Then(native, rest..., translated...); - }; - - template - static const auto fn() { - return next_step::template fn>(); + EOS_VM_FROM_WASM(memcpy_params, (vm::wasm_ptr_t dst, vm::wasm_ptr_t src, vm::wasm_size_t size)) { + auto d = array_ptr_impl(dst, size); + auto s = array_ptr_impl(src, size); + array_ptr_impl(dst, 1); + return { d, s, size }; } -}; -/** - * Specialization for transcribing a array_ptr type in the native method signature - * This type transcribes into 2 wasm parameters: a pointer and byte length and checks the validity of that memory - * range before dispatching to the native method - * - * @tparam Ret - the return type of the native method - * @tparam Inputs - the remaining native parameters to transcribe - * @tparam Translated - the list of transcribed wasm parameters - */ -template -struct intrinsic_invoker_impl, uint32_t, Inputs...>, std::tuple> { - using next_step = intrinsic_invoker_impl, std::tuple>; - using then_type = Ret(*)(array_ptr, uint32_t, Inputs..., Translated...); - - template - static auto translate_one(Inputs... rest, Translated... translated, I32 ptr, I32 size) -> std::enable_if_t::value, Ret> { - static_assert(!std::is_pointer::value, "Currently don't support array of pointers"); - const auto length = size_t((U32)size); - T* base = array_ptr_impl((U32)ptr, length); - if ( reinterpret_cast(base) % alignof(T) != 0 ) { - std::vector& copy = reinterpret_cast>*>(eos_vm_oc_get_bounce_buffer_list())->emplace_back(length > 0 ? length*sizeof(T) : 1); - T* copy_ptr = (T*)©[0]; - memcpy( (void*)copy.data(), (void*)base, length * sizeof(T) ); - return Then(static_cast>(copy_ptr), length, rest..., translated...); - } - return Then(static_cast>(base), length, rest..., translated...); - }; - - template - static auto translate_one(Inputs... rest, Translated... translated, I32 ptr, I32 size) -> std::enable_if_t::value, Ret> { - static_assert(!std::is_pointer::value, "Currently don't support array of pointers"); - const auto length = size_t((U32)size); - T* base = array_ptr_impl((U32)ptr, length); - if ( reinterpret_cast(base) % alignof(T) != 0 ) { - std::vector& copy = reinterpret_cast>*>(eos_vm_oc_get_bounce_buffer_list())->emplace_back(length > 0 ? length*sizeof(T) : 1); - T* copy_ptr = (T*)©[0]; - memcpy( (void*)copy.data(), (void*)base, length * sizeof(T) ); - Ret ret = Then(static_cast>(copy_ptr), length, rest..., translated...); - memcpy( (void*)base, (void*)copy.data(), length * sizeof(T) ); - return ret; - } - return Then(static_cast>(base), length, rest..., translated...); - }; - - template - static const auto fn() { - return next_step::template fn>(); + EOS_VM_FROM_WASM(memcmp_params, (vm::wasm_ptr_t lhs, vm::wasm_ptr_t rhs, vm::wasm_size_t size)) { + auto l = array_ptr_impl(lhs, size); + auto r = array_ptr_impl(rhs, size); + return { l, r, size }; } -}; -/** - * Specialization for transcribing a null_terminated_ptr type in the native method signature - * This type transcribes 1 wasm parameters: a char pointer which is validated to contain - * a null value before the end of the allocated memory. - * - * @tparam Ret - the return type of the native method - * @tparam Inputs - the remaining native parameters to transcribe - * @tparam Translated - the list of transcribed wasm parameters - */ -template -struct intrinsic_invoker_impl, std::tuple> { - using next_step = intrinsic_invoker_impl, std::tuple>; - using then_type = Ret(*)(null_terminated_ptr, Inputs..., Translated...); - - template - static Ret translate_one(Inputs... rest, Translated... translated, I32 ptr) { - return Then(null_terminated_ptr_impl((U32)ptr), rest..., translated...); - }; - - template - static const auto fn() { - return next_step::template fn>(); + EOS_VM_FROM_WASM(memset_params, (vm::wasm_ptr_t dst, int32_t val, vm::wasm_size_t size)) { + auto d = array_ptr_impl(dst, size); + array_ptr_impl(dst, 1); + return { d, val, size }; } -}; -/** - * Specialization for transcribing a pair of array_ptr types in the native method signature that share size - * This type transcribes into 3 wasm parameters: 2 pointers and byte length and checks the validity of those memory - * ranges before dispatching to the native method - * - * @tparam Ret - the return type of the native method - * @tparam Inputs - the remaining native parameters to transcribe - * @tparam Translated - the list of transcribed wasm parameters - */ -template -struct intrinsic_invoker_impl, array_ptr, uint32_t, Inputs...>, std::tuple> { - using next_step = intrinsic_invoker_impl, std::tuple>; - using then_type = Ret(*)(array_ptr, array_ptr, uint32_t, Inputs..., Translated...); - - template - static Ret translate_one(Inputs... rest, Translated... translated, I32 ptr_t, I32 ptr_u, I32 size) { - static_assert(std::is_same, char>::value && std::is_same, char>::value, "Currently only support array of (const)chars"); - const auto length = size_t((U32)size); - return Then(array_ptr_impl((U32)ptr_t, length), array_ptr_impl((U32)ptr_u, length), length, rest..., translated...); - }; - - template - static const auto fn() { - return next_step::template fn>(); + template + auto from_wasm(vm::wasm_ptr_t ptr) const + -> std::enable_if_t< std::is_pointer_v, + vm::argument_proxy > { + void* p = array_ptr_impl>(ptr, 1); + return {p}; } -}; -/** - * Specialization for transcribing memset parameters - * - * @tparam Ret - the return type of the native method - * @tparam Inputs - the remaining native parameters to transcribe - * @tparam Translated - the list of transcribed wasm parameters - */ -template -struct intrinsic_invoker_impl, int, uint32_t>, std::tuple<>> { - using next_step = intrinsic_invoker_impl, std::tuple>; - using then_type = Ret(*)(array_ptr, int, uint32_t); - - template - static Ret translate_one(I32 ptr, I32 value, I32 size) { - const auto length = size_t((U32)size); - return Then(array_ptr_impl((U32)ptr, length), value, length); - }; - - template - static const auto fn() { - return next_step::template fn>(); + template + auto from_wasm(vm::wasm_ptr_t ptr, vm::wasm_size_t len, vm::tag = {}) const + -> std::enable_if_t, T> { + void* p = array_ptr_impl(ptr, len); + return {static_cast(p), len}; } -}; - -/** - * Specialization for transcribing a pointer type in the native method signature - * This type transcribes into an int32 pointer checks the validity of that memory - * range before dispatching to the native method - * - * @tparam Ret - the return type of the native method - * @tparam Inputs - the remaining native parameters to transcribe - * @tparam Translated - the list of transcribed wasm parameters - */ -template -struct intrinsic_invoker_impl, std::tuple> { - using next_step = intrinsic_invoker_impl, std::tuple>; - using then_type = Ret (*)(T *, Inputs..., Translated...); - - template - static auto translate_one(Inputs... rest, Translated... translated, I32 ptr) -> std::enable_if_t::value, Ret> { - T* base = array_ptr_impl((U32)ptr, 1); - if ( reinterpret_cast(base) % alignof(T) != 0 ) { - std::remove_const_t copy; - T* copy_ptr = © - memcpy( (void*)copy_ptr, (void*)base, sizeof(T) ); - return Then(copy_ptr, rest..., translated...); - } - return Then(base, rest..., translated...); - }; - - template - static auto translate_one(Inputs... rest, Translated... translated, I32 ptr) -> std::enable_if_t::value, Ret> { - T* base = array_ptr_impl((U32)ptr, 1); - if ( reinterpret_cast(base) % alignof(T) != 0 ) { - std::remove_const_t copy; - T* copy_ptr = © - memcpy( (void*)copy_ptr, (void*)base, sizeof(T) ); - Ret ret = Then(copy_ptr, rest..., translated...); - memcpy( (void*)base, (void*)copy_ptr, sizeof(T) ); - return ret; - } - return Then(base, rest..., translated...); - }; - template - static const auto fn() { - return next_step::template fn>(); + template + auto from_wasm(vm::wasm_ptr_t ptr, vm::wasm_size_t len, vm::tag = {}) const + -> std::enable_if_t< vm::is_argument_proxy_type_v && + vm::is_span_type_v, T> { + void* p = array_ptr_impl(ptr, len); + return {p, len}; } -}; -/** - * Specialization for transcribing a reference to a name which can be passed as a native value - * This type transcribes into a native type which is loaded by value into a - * variable on the stack and then passed by reference to the intrinsic. - * - * @tparam Ret - the return type of the native method - * @tparam Inputs - the remaining native parameters to transcribe - * @tparam Translated - the list of transcribed wasm parameters - */ -template -struct intrinsic_invoker_impl, std::tuple> { - using next_step = intrinsic_invoker_impl, std::tuple >>; - using then_type = Ret (*)(const name&, Inputs..., Translated...); - - template - static Ret translate_one(Inputs... rest, Translated... translated, native_to_wasm_t wasm_value) { - auto value = name(wasm_value); - return Then(value, rest..., translated...); - } - - template - static const auto fn() { - return next_step::template fn>(); - } -}; - -/** - * Specialization for transcribing a reference type in the native method signature - * This type transcribes into an int32 pointer checks the validity of that memory - * range before dispatching to the native method - * - * @tparam Ret - the return type of the native method - * @tparam Inputs - the remaining native parameters to transcribe - * @tparam Translated - the list of transcribed wasm parameters - */ -template -struct intrinsic_invoker_impl, std::tuple> { - using next_step = intrinsic_invoker_impl, std::tuple>; - using then_type = Ret (*)(T &, Inputs..., Translated...); - - template - static auto translate_one(Inputs... rest, Translated... translated, I32 ptr) -> std::enable_if_t::value, Ret> { - EOS_ASSERT((U32)ptr != 0, wasm_exception, "references cannot be created for null pointers"); - T &base = *array_ptr_impl((uint32_t)ptr, 1); - - if ( reinterpret_cast(&base) % alignof(T) != 0 ) { - std::remove_const_t copy; - T* copy_ptr = © - memcpy( (void*)copy_ptr, (void*)&base, sizeof(T) ); - return Then(*copy_ptr, rest..., translated...); + template + auto from_wasm(vm::wasm_ptr_t ptr, vm::tag = {}) const + -> std::enable_if_t< vm::is_argument_proxy_type_v && + std::is_pointer_v, T> { + if constexpr(T::is_legacy()) { + EOS_ASSERT(ptr != 0, wasm_execution_error, "references cannot be created for null pointers"); } - return Then(base, rest..., translated...); + void* p = array_ptr_impl(ptr, 1); + return {p}; } - template - static auto translate_one(Inputs... rest, Translated... translated, I32 ptr) -> std::enable_if_t::value, Ret> { - EOS_ASSERT((U32)ptr != 0, wasm_exception, "reference cannot be created for null pointers"); - T &base = *array_ptr_impl((uint32_t)ptr, 1); - - if ( reinterpret_cast(&base) % alignof(T) != 0 ) { - std::remove_const_t copy; - T* copy_ptr = © - memcpy( (void*)copy_ptr, (void*)&base, sizeof(T) ); - Ret ret = Then(*copy_ptr, rest..., translated...); - memcpy( (void*)&base, (void*)copy_ptr, sizeof(T) ); - return ret; - } - return Then(base, rest..., translated...); + EOS_VM_FROM_WASM(null_terminated_ptr, (vm::wasm_ptr_t ptr)) { + auto p = null_terminated_ptr_impl(ptr); + return {static_cast(p)}; } - - template - static const auto fn() { - return next_step::template fn>(); + EOS_VM_FROM_WASM(name, (uint64_t e)) { return name{e}; } + uint64_t to_wasm(name&& n) { return n.to_uint64_t(); } + vm::wasm_ptr_t to_wasm(void*&& ptr) { + return convert_native_to_wasm(static_cast(ptr)); + } + EOS_VM_FROM_WASM(float32_t, (float f)) { return ::to_softfloat32(f); } + EOS_VM_FROM_WASM(float64_t, (double f)) { return ::to_softfloat64(f); } + + template + inline decltype(auto) as_value(const vm::native_value& val) const { + if constexpr (std::is_integral_v && sizeof(T) == 4) + return static_cast(val.i32); + else if constexpr (std::is_integral_v && sizeof(T) == 8) + return static_cast(val.i64); + else if constexpr (std::is_floating_point_v && sizeof(T) == 4) + return static_cast(val.f32); + else if constexpr (std::is_floating_point_v && sizeof(T) == 8) + return static_cast(val.f64); + // No direct pointer support + else + return vm::no_match_t{}; } }; -/** - * forward declaration of a wrapper class to call methods of the class - */ -template -struct intrinsic_function_invoker { - using impl = intrinsic_invoker_impl, std::tuple<>>; +template +auto get_ct_args(std::index_sequence); - template - static Ret wrapper(Params... params) { - constexpr int cb_ctx_ptr_offset = OFFSET_OF_CONTROL_BLOCK_MEMBER(ctx); - apply_context* ctx; - asm("mov %%gs:%c[applyContextOffset], %[cPtr]\n" - : [cPtr] "=r" (ctx) - : [applyContextOffset] "i" (cb_ctx_ptr_offset) - ); - return (class_from_wasm::value(*ctx).*Method)(params...); - } +inline uint32_t make_native_type(vm::i32_const_t x) { return x.data.ui; } +inline uint64_t make_native_type(vm::i64_const_t x) { return x.data.ui; } +inline float make_native_type(vm::f32_const_t x) { return x.data.f; } +inline double make_native_type(vm::f64_const_t x) { return x.data.f; } - template - static const WasmSig *fn() { - auto fn = impl::template fn>(); - static_assert(std::is_same::value, - "Intrinsic function signature does not match the ABI"); - return fn; +template +auto get_ct_args_one(std::index_sequence) { + return std::tuple().as_result(std::declval>())))...>(); +} + +template +auto get_ct_args_i() { + if constexpr (vm::detail::has_from_wasm_v) { + using args_tuple = vm::detail::from_wasm_type_deducer_t; + return get_ct_args_one(std::make_index_sequence>()); + } else { + return std::tuple().as_result(std::declval())))>(); } -}; +} -template -struct intrinsic_function_invoker { - using impl = intrinsic_invoker_impl, std::tuple<>>; +template +auto get_ct_args(std::index_sequence) { + return std::tuple_cat(get_ct_args_i>()...); +} - template - static void_type wrapper(Params... params) { +struct result_resolver { + // Suppress "expression result unused" warnings + result_resolver(eos_vm_oc_type_converter& tc) : tc(tc) {} + template + auto operator,(T&& res) { + return make_native_type(vm::detail::resolve_result(tc, static_cast(res))); + } + eos_vm_oc_type_converter& tc; +}; + +template +auto fn(A... a) { + try { + if constexpr(!is_injected) { + constexpr int cb_current_call_depth_remaining_segment_offset = OFFSET_OF_CONTROL_BLOCK_MEMBER(current_call_depth_remaining); + constexpr int depth_assertion_intrinsic_offset = OFFSET_OF_FIRST_INTRINSIC - (int) find_intrinsic_index("eosvmoc_internal.depth_assert") * 8; + + asm volatile("cmpl $1,%%gs:%c[callDepthRemainOffset]\n" + "jne 1f\n" + "callq *%%gs:%c[depthAssertionIntrinsicOffset]\n" + "1:\n" + : + : [callDepthRemainOffset] "i" (cb_current_call_depth_remaining_segment_offset), + [depthAssertionIntrinsicOffset] "i" (depth_assertion_intrinsic_offset) + : "cc"); + } + using native_args = vm::flatten_parameters_t; + eosio::vm::native_value stack[] = { a... }; constexpr int cb_ctx_ptr_offset = OFFSET_OF_CONTROL_BLOCK_MEMBER(ctx); apply_context* ctx; asm("mov %%gs:%c[applyContextOffset], %[cPtr]\n" - : [cPtr] "=r" (ctx) - : [applyContextOffset] "i" (cb_ctx_ptr_offset) - ); - (class_from_wasm::value(*ctx).*Method)(params...); - return void_type(); + : [cPtr] "=r" (ctx) + : [applyContextOffset] "i" (cb_ctx_ptr_offset) + ); + Interface host(*ctx); + eos_vm_oc_type_converter tc{&host, eos_vm_oc_execution_interface{stack + sizeof...(A)}}; + return result_resolver{tc}, eosio::vm::invoke_with_host(tc, &host, std::make_index_sequence()); } - - template - static const WasmSig *fn() { - auto fn = impl::template fn>(); - static_assert(std::is_same::value, - "Intrinsic function signature does not match the ABI"); - return fn; + catch(...) { + *reinterpret_cast(eos_vm_oc_get_exception_ptr()) = std::current_exception(); } -}; - -template -struct intrinsic_function_invoker_wrapper; - -template -struct intrinsic_function_invoker_wrapper { - static_assert( !(std::is_pointer_v && alignof(std::remove_pointer_t>) != 1) && - !(std::is_lvalue_reference_v && alignof(std::remove_reference_t>) != 1), - "intrinsics should only return a reference or pointer with single byte alignment"); - using type = intrinsic_function_invoker; -}; - -template -struct intrinsic_function_invoker_wrapper { - static_assert( !(std::is_pointer_v && alignof(std::remove_pointer_t>) != 1) && - !(std::is_lvalue_reference_v && alignof(std::remove_reference_t>) != 1), - "intrinsics should only return a reference or pointer with single byte alignment"); - using type = intrinsic_function_invoker; -}; - -template -struct intrinsic_function_invoker_wrapper { - static_assert( !(std::is_pointer_v && alignof(std::remove_pointer_t>) != 1) && - !(std::is_lvalue_reference_v && alignof(std::remove_reference_t>) != 1), - "intrinsics should only return a reference or pointer with single byte alignment"); - using type = intrinsic_function_invoker; -}; - -template -struct intrinsic_function_invoker_wrapper { - static_assert( !(std::is_pointer_v && alignof(std::remove_pointer_t>) != 1) && - !(std::is_lvalue_reference_v && alignof(std::remove_reference_t>) != 1), - "intrinsics should only return a reference or pointer with single byte alignment"); - using type = intrinsic_function_invoker; -}; - -#define _ADD_PAREN_1(...) ((__VA_ARGS__)) _ADD_PAREN_2 -#define _ADD_PAREN_2(...) ((__VA_ARGS__)) _ADD_PAREN_1 -#define _ADD_PAREN_1_END -#define _ADD_PAREN_2_END -#define _WRAPPED_SEQ(SEQ) BOOST_PP_CAT(_ADD_PAREN_1 SEQ, _END) + siglongjmp(*eos_vm_oc_get_jmp_buf(), EOSVMOC_EXIT_EXCEPTION); + __builtin_unreachable(); +} -#define __INTRINSIC_NAME(LABEL, SUFFIX) LABEL##SUFFIX -#define _INTRINSIC_NAME(LABEL, SUFFIX) __INTRINSIC_NAME(LABEL,SUFFIX) +template +constexpr auto create_function(std::index_sequence) { + return &fn...>; +} -#define _REGISTER_EOSVMOC_INTRINSIC(CLS, MOD, METHOD, WASM_SIG, NAME, SIG)\ - static eosio::chain::eosvmoc::intrinsic _INTRINSIC_NAME(__intrinsic_fn, __COUNTER__) EOSVMOC_INTRINSIC_INIT_PRIORITY (\ - MOD "." NAME,\ - eosio::chain::webassembly::eosvmoc::wasm_function_type_provider::type(),\ - (void *)eosio::chain::webassembly::eosvmoc::intrinsic_function_invoker_wrapper::type::fn<&CLS::METHOD>(),\ - ::boost::hana::index_if(eosio::chain::eosvmoc::intrinsic_table, ::boost::hana::equal.to(BOOST_HANA_STRING(MOD "." NAME))).value()\ - );\ +template +constexpr auto create_function() { + using native_args = vm::flatten_parameters_t; + using wasm_args = decltype(get_ct_args(std::make_index_sequence>())); + return create_function(std::make_index_sequence>()); +} +template +void register_eosvm_oc(Name n) { + // Has special handling + if(n == BOOST_HANA_STRING("env.eosio_exit")) return; + constexpr auto fn = create_function(); + constexpr auto index = find_intrinsic_index(n.c_str()); + intrinsic the_intrinsic( + n.c_str(), + wasm_function_type_provider>::type(), + reinterpret_cast(fn), + index + ); +} } } } }// eosio::chain::webassembly::eosvmoc diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/code_cache.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/code_cache.hpp index ce94cc5db90..2f5031d3cf7 100644 --- a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/code_cache.hpp +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/code_cache.hpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include #include @@ -52,7 +52,7 @@ class code_cache_base { code_descriptor, indexed_by< sequenced<>, - ordered_unique, + hashed_unique, composite_key< code_descriptor, member, member diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/eos-vm-oc.h b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/eos-vm-oc.h index 6c5d4e42676..28a8bd962e7 100644 --- a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/eos-vm-oc.h +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/eos-vm-oc.h @@ -35,4 +35,6 @@ struct eos_vm_oc_control_block { uintptr_t running_code_base; int64_t first_invalid_memory_address; unsigned is_running; -}; \ No newline at end of file + int64_t max_linear_memory_pages; + void* globals; +}; diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/eos-vm-oc.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/eos-vm-oc.hpp index 6dfffa12804..d378294e9ea 100644 --- a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/eos-vm-oc.hpp +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/eos-vm-oc.hpp @@ -21,10 +21,14 @@ namespace eosvmoc { using control_block = eos_vm_oc_control_block; struct no_offset{}; -struct code_offset{ size_t offset; }; //relative to code_begin -struct intrinsic_ordinal{ size_t ordinal; }; +struct code_offset { + size_t offset; +}; +struct intrinsic_ordinal { + size_t ordinal; +}; -using eosvmoc_optional_offset_or_import_t = fc::static_variant; +using eosvmoc_optional_offset_or_import_t = std::variant; struct code_descriptor { digest_type code_hash; @@ -53,4 +57,4 @@ FC_REFLECT(eosio::chain::eosvmoc::code_offset, (offset)); FC_REFLECT(eosio::chain::eosvmoc::intrinsic_ordinal, (ordinal)); FC_REFLECT(eosio::chain::eosvmoc::code_descriptor, (code_hash)(vm_version)(codegen_version)(code_begin)(start)(apply_offset)(starting_memory_pages)(initdata_begin)(initdata_size)(initdata_prologue_size)); -#define EOSVMOC_INTRINSIC_INIT_PRIORITY __attribute__((init_priority(198))) \ No newline at end of file +#define EOSVMOC_INTRINSIC_INIT_PRIORITY __attribute__((init_priority(198))) diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/executor.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/executor.hpp index e5a092cd57e..5050a3eee43 100644 --- a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/executor.hpp +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/executor.hpp @@ -1,5 +1,7 @@ #pragma once +#include + #include #include #include @@ -24,7 +26,7 @@ class executor { executor(const code_cache_base& cc); ~executor(); - void execute(const code_descriptor& code, const memory& mem, apply_context& context); + void execute(const code_descriptor& code, memory& mem, apply_context& context); private: uint8_t* code_mapping; @@ -34,6 +36,8 @@ class executor { std::exception_ptr executors_exception_ptr; sigjmp_buf executors_sigjmp_buf; std::list> executors_bounce_buffers; + std::vector globals_buffer; + execution_stack stack; }; }}} diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/gs_seg_helpers.h b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/gs_seg_helpers.h index d2baf938017..36d7e891a58 100644 --- a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/gs_seg_helpers.h +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/gs_seg_helpers.h @@ -16,7 +16,7 @@ //the values ever slide which would be a PIC breaking event we'd want to know about at compile //time. #define EOS_VM_OC_CONTROL_BLOCK_OFFSET (-18944) -#define EOS_VM_OC_MEMORY_STRIDE (UINT64_C(4329598976)) +#define EOS_VM_OC_MEMORY_STRIDE (UINT64_C(8589963264)) #ifdef __cplusplus extern "C" { @@ -29,4 +29,4 @@ void* eos_vm_oc_get_bounce_buffer_list(); #ifdef __cplusplus } -#endif \ No newline at end of file +#endif diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/intrinsic_mapping.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/intrinsic_mapping.hpp index 022df920918..fdf5d162ce8 100644 --- a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/intrinsic_mapping.hpp +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/intrinsic_mapping.hpp @@ -1,254 +1,294 @@ #pragma once -#define BOOST_HANA_CONFIG_ENABLE_STRING_UDL -#include -#include +#include +#include +#include namespace eosio { namespace chain { namespace eosvmoc { - -using namespace boost::hana::literals; - //NEVER reorder or remove indexes; the PIC uses the indexes in this table as an offset in to a jump // table. Adding on the bottom is fine and requires no other updates elsewhere -constexpr auto intrinsic_table = boost::hana::make_tuple( - "eosvmoc_internal.unreachable"_s, - "eosvmoc_internal.grow_memory"_s, - "eosvmoc_internal.div0_or_overflow"_s, - "eosvmoc_internal.indirect_call_mismatch"_s, - "eosvmoc_internal.indirect_call_oob"_s, - "eosvmoc_internal.depth_assert"_s, - "eosio_injection.call_depth_assert"_s, //these two are never used by EOS VM OC but all intrinsics - "eosio_injection.checktime"_s, //must be mapped - "env.__ashlti3"_s, - "env.__ashrti3"_s, - "env.__lshlti3"_s, - "env.__lshrti3"_s, - "env.__divti3"_s, - "env.__udivti3"_s, - "env.__modti3"_s, - "env.__umodti3"_s, - "env.__multi3"_s, - "env.__addtf3"_s, - "env.__subtf3"_s, - "env.__multf3"_s, - "env.__divtf3"_s, - "env.__eqtf2"_s, - "env.__netf2"_s, - "env.__getf2"_s, - "env.__gttf2"_s, - "env.__lttf2"_s, - "env.__letf2"_s, - "env.__cmptf2"_s, - "env.__unordtf2"_s, - "env.__negtf2"_s, - "env.__floatsitf"_s, - "env.__floatunsitf"_s, - "env.__floatditf"_s, - "env.__floatunditf"_s, - "env.__floattidf"_s, - "env.__floatuntidf"_s, - "env.__floatsidf"_s, - "env.__extendsftf2"_s, - "env.__extenddftf2"_s, - "env.__fixtfti"_s, - "env.__fixtfdi"_s, - "env.__fixtfsi"_s, - "env.__fixunstfti"_s, - "env.__fixunstfdi"_s, - "env.__fixunstfsi"_s, - "env.__fixsfti"_s, - "env.__fixdfti"_s, - "env.__fixunssfti"_s, - "env.__fixunsdfti"_s, - "env.__trunctfdf2"_s, - "env.__trunctfsf2"_s, - "env.is_feature_active"_s, - "env.activate_feature"_s, - "env.get_resource_limits"_s, - "env.set_resource_limits"_s, - "env.set_proposed_producers"_s, - "env.set_proposed_producers_ex"_s, - "env.get_blockchain_parameters_packed"_s, - "env.set_blockchain_parameters_packed"_s, - "env.is_privileged"_s, - "env.set_privileged"_s, - "env.preactivate_feature"_s, - "env.get_active_producers"_s, - "env.db_store_i64"_s, - "env.db_update_i64"_s, - "env.db_remove_i64"_s, - "env.db_get_i64"_s, - "env.db_next_i64"_s, - "env.db_previous_i64"_s, - "env.db_find_i64"_s, - "env.db_lowerbound_i64"_s, - "env.db_upperbound_i64"_s, - "env.db_end_i64"_s, - "env.db_idx64_store"_s, - "env.db_idx64_remove"_s, - "env.db_idx64_update"_s, - "env.db_idx64_find_primary"_s, - "env.db_idx64_find_secondary"_s, - "env.db_idx64_lowerbound"_s, - "env.db_idx64_upperbound"_s, - "env.db_idx64_end"_s, - "env.db_idx64_next"_s, - "env.db_idx64_previous"_s, - "env.db_idx128_store"_s, - "env.db_idx128_remove"_s, - "env.db_idx128_update"_s, - "env.db_idx128_find_primary"_s, - "env.db_idx128_find_secondary"_s, - "env.db_idx128_lowerbound"_s, - "env.db_idx128_upperbound"_s, - "env.db_idx128_end"_s, - "env.db_idx128_next"_s, - "env.db_idx128_previous"_s, - "env.db_idx_double_store"_s, - "env.db_idx_double_remove"_s, - "env.db_idx_double_update"_s, - "env.db_idx_double_find_primary"_s, - "env.db_idx_double_find_secondary"_s, - "env.db_idx_double_lowerbound"_s, - "env.db_idx_double_upperbound"_s, - "env.db_idx_double_end"_s, - "env.db_idx_double_next"_s, - "env.db_idx_double_previous"_s, - "env.db_idx_long_double_store"_s, - "env.db_idx_long_double_remove"_s, - "env.db_idx_long_double_update"_s, - "env.db_idx_long_double_find_primary"_s, - "env.db_idx_long_double_find_secondary"_s, - "env.db_idx_long_double_lowerbound"_s, - "env.db_idx_long_double_upperbound"_s, - "env.db_idx_long_double_end"_s, - "env.db_idx_long_double_next"_s, - "env.db_idx_long_double_previous"_s, - "env.db_idx256_store"_s, - "env.db_idx256_remove"_s, - "env.db_idx256_update"_s, - "env.db_idx256_find_primary"_s, - "env.db_idx256_find_secondary"_s, - "env.db_idx256_lowerbound"_s, - "env.db_idx256_upperbound"_s, - "env.db_idx256_end"_s, - "env.db_idx256_next"_s, - "env.db_idx256_previous"_s, - "env.assert_recover_key"_s, - "env.recover_key"_s, - "env.assert_sha256"_s, - "env.assert_sha1"_s, - "env.assert_sha512"_s, - "env.assert_ripemd160"_s, - "env.sha1"_s, - "env.sha256"_s, - "env.sha512"_s, - "env.ripemd160"_s, - "env.check_transaction_authorization"_s, - "env.check_permission_authorization"_s, - "env.get_permission_last_used"_s, - "env.get_account_creation_time"_s, - "env.current_time"_s, - "env.publication_time"_s, - "env.is_feature_activated"_s, - "env.get_sender"_s, - "env.abort"_s, - "env.eosio_assert"_s, - "env.eosio_assert_message"_s, - "env.eosio_assert_code"_s, - "env.eosio_exit"_s, - "env.read_action_data"_s, - "env.action_data_size"_s, - "env.current_receiver"_s, - "env.require_recipient"_s, - "env.require_auth"_s, - "env.require_auth2"_s, - "env.has_auth"_s, - "env.is_account"_s, - "env.prints"_s, - "env.prints_l"_s, - "env.printi"_s, - "env.printui"_s, - "env.printi128"_s, - "env.printui128"_s, - "env.printsf"_s, - "env.printdf"_s, - "env.printqf"_s, - "env.printn"_s, - "env.printhex"_s, - "env.read_transaction"_s, - "env.transaction_size"_s, - "env.expiration"_s, - "env.tapos_block_prefix"_s, - "env.tapos_block_num"_s, - "env.get_action"_s, - "env.send_inline"_s, - "env.send_context_free_inline"_s, - "env.send_deferred"_s, - "env.cancel_deferred"_s, - "env.get_context_free_data"_s, - "env.memcpy"_s, - "env.memmove"_s, - "env.memcmp"_s, - "env.memset"_s, - "eosio_injection._eosio_f32_add"_s, - "eosio_injection._eosio_f32_sub"_s, - "eosio_injection._eosio_f32_mul"_s, - "eosio_injection._eosio_f32_div"_s, - "eosio_injection._eosio_f32_min"_s, - "eosio_injection._eosio_f32_max"_s, - "eosio_injection._eosio_f32_copysign"_s, - "eosio_injection._eosio_f32_abs"_s, - "eosio_injection._eosio_f32_neg"_s, - "eosio_injection._eosio_f32_sqrt"_s, - "eosio_injection._eosio_f32_ceil"_s, - "eosio_injection._eosio_f32_floor"_s, - "eosio_injection._eosio_f32_trunc"_s, - "eosio_injection._eosio_f32_nearest"_s, - "eosio_injection._eosio_f32_eq"_s, - "eosio_injection._eosio_f32_ne"_s, - "eosio_injection._eosio_f32_lt"_s, - "eosio_injection._eosio_f32_le"_s, - "eosio_injection._eosio_f32_gt"_s, - "eosio_injection._eosio_f32_ge"_s, - "eosio_injection._eosio_f64_add"_s, - "eosio_injection._eosio_f64_sub"_s, - "eosio_injection._eosio_f64_mul"_s, - "eosio_injection._eosio_f64_div"_s, - "eosio_injection._eosio_f64_min"_s, - "eosio_injection._eosio_f64_max"_s, - "eosio_injection._eosio_f64_copysign"_s, - "eosio_injection._eosio_f64_abs"_s, - "eosio_injection._eosio_f64_neg"_s, - "eosio_injection._eosio_f64_sqrt"_s, - "eosio_injection._eosio_f64_ceil"_s, - "eosio_injection._eosio_f64_floor"_s, - "eosio_injection._eosio_f64_trunc"_s, - "eosio_injection._eosio_f64_nearest"_s, - "eosio_injection._eosio_f64_eq"_s, - "eosio_injection._eosio_f64_ne"_s, - "eosio_injection._eosio_f64_lt"_s, - "eosio_injection._eosio_f64_le"_s, - "eosio_injection._eosio_f64_gt"_s, - "eosio_injection._eosio_f64_ge"_s, - "eosio_injection._eosio_f32_promote"_s, - "eosio_injection._eosio_f64_demote"_s, - "eosio_injection._eosio_f32_trunc_i32s"_s, - "eosio_injection._eosio_f64_trunc_i32s"_s, - "eosio_injection._eosio_f32_trunc_i32u"_s, - "eosio_injection._eosio_f64_trunc_i32u"_s, - "eosio_injection._eosio_f32_trunc_i64s"_s, - "eosio_injection._eosio_f64_trunc_i64s"_s, - "eosio_injection._eosio_f32_trunc_i64u"_s, - "eosio_injection._eosio_f64_trunc_i64u"_s, - "eosio_injection._eosio_i32_to_f32"_s, - "eosio_injection._eosio_i64_to_f32"_s, - "eosio_injection._eosio_ui32_to_f32"_s, - "eosio_injection._eosio_ui64_to_f32"_s, - "eosio_injection._eosio_i32_to_f64"_s, - "eosio_injection._eosio_i64_to_f64"_s, - "eosio_injection._eosio_ui32_to_f64"_s, - "eosio_injection._eosio_ui64_to_f64"_s -); +namespace detail { + template + inline constexpr auto generate_table( Args&&... args ) { + return std::array { args... }; + } +} // ns eosio::chain::eosvmoc::detail + +inline constexpr auto get_intrinsic_table() { + return detail::generate_table( + "eosvmoc_internal.unreachable", + "eosvmoc_internal.grow_memory", + "eosvmoc_internal.div0_or_overflow", + "eosvmoc_internal.indirect_call_mismatch", + "eosvmoc_internal.indirect_call_oob", + "eosvmoc_internal.depth_assert", + "eosio_injection.call_depth_assert", //these two are never used by EOS VM OC but all intrinsics + "eosio_injection.checktime", //must be mapped + "env.__ashlti3", + "env.__ashrti3", + "env.__lshlti3", + "env.__lshrti3", + "env.__divti3", + "env.__udivti3", + "env.__modti3", + "env.__umodti3", + "env.__multi3", + "env.__addtf3", + "env.__subtf3", + "env.__multf3", + "env.__divtf3", + "env.__eqtf2", + "env.__netf2", + "env.__getf2", + "env.__gttf2", + "env.__lttf2", + "env.__letf2", + "env.__cmptf2", + "env.__unordtf2", + "env.__negtf2", + "env.__floatsitf", + "env.__floatunsitf", + "env.__floatditf", + "env.__floatunditf", + "env.__floattidf", + "env.__floatuntidf", + "env.__floatsidf", + "env.__extendsftf2", + "env.__extenddftf2", + "env.__fixtfti", + "env.__fixtfdi", + "env.__fixtfsi", + "env.__fixunstfti", + "env.__fixunstfdi", + "env.__fixunstfsi", + "env.__fixsfti", + "env.__fixdfti", + "env.__fixunssfti", + "env.__fixunsdfti", + "env.__trunctfdf2", + "env.__trunctfsf2", + "env.is_feature_active", + "env.activate_feature", + "env.get_resource_limits", + "env.set_resource_limits", + "env.set_proposed_producers", + "env.set_proposed_producers_ex", + "env.get_blockchain_parameters_packed", + "env.set_blockchain_parameters_packed", + "env.is_privileged", + "env.set_privileged", + "env.preactivate_feature", + "env.get_active_producers", + "env.db_store_i64", + "env.db_update_i64", + "env.db_remove_i64", + "env.db_get_i64", + "env.db_next_i64", + "env.db_previous_i64", + "env.db_find_i64", + "env.db_lowerbound_i64", + "env.db_upperbound_i64", + "env.db_end_i64", + "env.db_idx64_store", + "env.db_idx64_remove", + "env.db_idx64_update", + "env.db_idx64_find_primary", + "env.db_idx64_find_secondary", + "env.db_idx64_lowerbound", + "env.db_idx64_upperbound", + "env.db_idx64_end", + "env.db_idx64_next", + "env.db_idx64_previous", + "env.db_idx128_store", + "env.db_idx128_remove", + "env.db_idx128_update", + "env.db_idx128_find_primary", + "env.db_idx128_find_secondary", + "env.db_idx128_lowerbound", + "env.db_idx128_upperbound", + "env.db_idx128_end", + "env.db_idx128_next", + "env.db_idx128_previous", + "env.db_idx_double_store", + "env.db_idx_double_remove", + "env.db_idx_double_update", + "env.db_idx_double_find_primary", + "env.db_idx_double_find_secondary", + "env.db_idx_double_lowerbound", + "env.db_idx_double_upperbound", + "env.db_idx_double_end", + "env.db_idx_double_next", + "env.db_idx_double_previous", + "env.db_idx_long_double_store", + "env.db_idx_long_double_remove", + "env.db_idx_long_double_update", + "env.db_idx_long_double_find_primary", + "env.db_idx_long_double_find_secondary", + "env.db_idx_long_double_lowerbound", + "env.db_idx_long_double_upperbound", + "env.db_idx_long_double_end", + "env.db_idx_long_double_next", + "env.db_idx_long_double_previous", + "env.db_idx256_store", + "env.db_idx256_remove", + "env.db_idx256_update", + "env.db_idx256_find_primary", + "env.db_idx256_find_secondary", + "env.db_idx256_lowerbound", + "env.db_idx256_upperbound", + "env.db_idx256_end", + "env.db_idx256_next", + "env.db_idx256_previous", + "env.assert_recover_key", + "env.recover_key", + "env.assert_sha256", + "env.assert_sha1", + "env.assert_sha512", + "env.assert_ripemd160", + "env.sha1", + "env.sha256", + "env.sha512", + "env.ripemd160", + "env.check_transaction_authorization", + "env.check_permission_authorization", + "env.get_permission_last_used", + "env.get_account_creation_time", + "env.current_time", + "env.publication_time", + "env.is_feature_activated", + "env.get_sender", + "env.abort", + "env.eosio_assert", + "env.eosio_assert_message", + "env.eosio_assert_code", + "env.eosio_exit", + "env.read_action_data", + "env.action_data_size", + "env.current_receiver", + "env.require_recipient", + "env.require_auth", + "env.require_auth2", + "env.has_auth", + "env.is_account", + "env.prints", + "env.prints_l", + "env.printi", + "env.printui", + "env.printi128", + "env.printui128", + "env.printsf", + "env.printdf", + "env.printqf", + "env.printn", + "env.printhex", + "env.read_transaction", + "env.transaction_size", + "env.expiration", + "env.tapos_block_prefix", + "env.tapos_block_num", + "env.get_action", + "env.send_inline", + "env.send_context_free_inline", + "env.send_deferred", + "env.cancel_deferred", + "env.get_context_free_data", + "env.memcpy", + "env.memmove", + "env.memcmp", + "env.memset", + "eosio_injection._eosio_f32_add", + "eosio_injection._eosio_f32_sub", + "eosio_injection._eosio_f32_mul", + "eosio_injection._eosio_f32_div", + "eosio_injection._eosio_f32_min", + "eosio_injection._eosio_f32_max", + "eosio_injection._eosio_f32_copysign", + "eosio_injection._eosio_f32_abs", + "eosio_injection._eosio_f32_neg", + "eosio_injection._eosio_f32_sqrt", + "eosio_injection._eosio_f32_ceil", + "eosio_injection._eosio_f32_floor", + "eosio_injection._eosio_f32_trunc", + "eosio_injection._eosio_f32_nearest", + "eosio_injection._eosio_f32_eq", + "eosio_injection._eosio_f32_ne", + "eosio_injection._eosio_f32_lt", + "eosio_injection._eosio_f32_le", + "eosio_injection._eosio_f32_gt", + "eosio_injection._eosio_f32_ge", + "eosio_injection._eosio_f64_add", + "eosio_injection._eosio_f64_sub", + "eosio_injection._eosio_f64_mul", + "eosio_injection._eosio_f64_div", + "eosio_injection._eosio_f64_min", + "eosio_injection._eosio_f64_max", + "eosio_injection._eosio_f64_copysign", + "eosio_injection._eosio_f64_abs", + "eosio_injection._eosio_f64_neg", + "eosio_injection._eosio_f64_sqrt", + "eosio_injection._eosio_f64_ceil", + "eosio_injection._eosio_f64_floor", + "eosio_injection._eosio_f64_trunc", + "eosio_injection._eosio_f64_nearest", + "eosio_injection._eosio_f64_eq", + "eosio_injection._eosio_f64_ne", + "eosio_injection._eosio_f64_lt", + "eosio_injection._eosio_f64_le", + "eosio_injection._eosio_f64_gt", + "eosio_injection._eosio_f64_ge", + "eosio_injection._eosio_f32_promote", + "eosio_injection._eosio_f64_demote", + "eosio_injection._eosio_f32_trunc_i32s", + "eosio_injection._eosio_f64_trunc_i32s", + "eosio_injection._eosio_f32_trunc_i32u", + "eosio_injection._eosio_f64_trunc_i32u", + "eosio_injection._eosio_f32_trunc_i64s", + "eosio_injection._eosio_f64_trunc_i64s", + "eosio_injection._eosio_f32_trunc_i64u", + "eosio_injection._eosio_f64_trunc_i64u", + "eosio_injection._eosio_i32_to_f32", + "eosio_injection._eosio_i64_to_f32", + "eosio_injection._eosio_ui32_to_f32", + "eosio_injection._eosio_ui64_to_f32", + "eosio_injection._eosio_i32_to_f64", + "eosio_injection._eosio_i64_to_f64", + "eosio_injection._eosio_ui32_to_f64", + "eosio_injection._eosio_ui64_to_f64", + "env.set_action_return_value", + "env.kv_erase", + "env.kv_set", + "env.kv_get", + "env.kv_get_data", + "env.kv_it_create", + "env.kv_it_destroy", + "env.kv_it_status", + "env.kv_it_compare", + "env.kv_it_key_compare", + "env.kv_it_move_to_end", + "env.kv_it_next", + "env.kv_it_prev", + "env.kv_it_lower_bound", + "env.kv_it_key", + "env.kv_it_value", + "env.get_resource_limit", + "env.set_resource_limit", + "env.get_kv_parameters_packed", + "env.set_kv_parameters_packed", + "env.get_wasm_parameters_packed", + "env.set_wasm_parameters_packed", + "env.get_parameters_packed", + "env.set_parameters_packed" + ); +} +inline constexpr std::size_t find_intrinsic_index(std::string_view hf) { + constexpr auto hosts = get_intrinsic_table(); + for ( std::size_t i = 0; i < hosts.size(); ++i ) + if ( hosts[i] == hf ) + return i; + return std::numeric_limits::max(); +} +inline constexpr std::size_t intrinsic_table_size() { + return std::tuple_size::value; +} }}} \ No newline at end of file diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/ipc_protocol.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/ipc_protocol.hpp index b38c3bce64b..2674a2dbd4d 100644 --- a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/ipc_protocol.hpp +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/ipc_protocol.hpp @@ -11,7 +11,7 @@ struct initialize_message { }; struct initalize_response_message { - fc::optional error_message; //no error message? everything groovy + std::optional error_message; //no error message? everything groovy }; struct code_tuple { @@ -41,9 +41,9 @@ struct code_compilation_result_message { struct compilation_result_unknownfailure {}; struct compilation_result_toofull {}; -using wasm_compilation_result = fc::static_variant; +using wasm_compilation_result = std::variant; struct wasm_compilation_result_message { code_tuple code; @@ -51,13 +51,12 @@ struct wasm_compilation_result_message { size_t cache_free_bytes; }; -using eosvmoc_message = fc::static_variant; +using eosvmoc_message = std::variant; }}} FC_REFLECT(eosio::chain::eosvmoc::initialize_message, ) @@ -68,4 +67,4 @@ FC_REFLECT(eosio::chain::eosvmoc::evict_wasms_message, (codes)) FC_REFLECT(eosio::chain::eosvmoc::code_compilation_result_message, (start)(apply_offset)(starting_memory_pages)(initdata_prologue_size)) FC_REFLECT(eosio::chain::eosvmoc::compilation_result_unknownfailure, ) FC_REFLECT(eosio::chain::eosvmoc::compilation_result_toofull, ) -FC_REFLECT(eosio::chain::eosvmoc::wasm_compilation_result_message, (code)(result)(cache_free_bytes)) \ No newline at end of file +FC_REFLECT(eosio::chain::eosvmoc::wasm_compilation_result_message, (code)(result)(cache_free_bytes)) diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/memory.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/memory.hpp index 509e183bd80..d329b830c07 100644 --- a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/memory.hpp +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/memory.hpp @@ -11,8 +11,7 @@ namespace eosio { namespace chain { namespace eosvmoc { class memory { - static constexpr uint64_t wasm_memory_size = eosio::chain::wasm_constraints::maximum_linear_memory; - static constexpr uint64_t intrinsic_count = boost::hana::length(intrinsic_table); + static constexpr uint64_t intrinsic_count = intrinsic_table_size(); //warning: changing the following 3 params will invalidate existing PIC static constexpr uint64_t mutable_global_size = 8u * eosio::chain::wasm_constraints::maximum_mutable_globals/4u; static constexpr uint64_t table_size = 16u * eosio::chain::wasm_constraints::maximum_table_elements; @@ -21,14 +20,15 @@ class memory { //round up the prologue to multiple of 4K page static constexpr uint64_t memory_prologue_size = ((memory::wcb_allowance + mutable_global_size + table_size + intrinsic_count*UINT64_C(8))+UINT64_C(4095))/UINT64_C(4096)*UINT64_C(4096); - //prologue + 33MB + 4GB fault buffer + 4096 addtional buffer for safety - static constexpr uint64_t total_memory_per_slice = memory_prologue_size + wasm_memory_size + UINT64_C(0x100000000) + UINT64_C(4096); - - static constexpr uint64_t number_slices = wasm_memory_size/(64u*1024u)+1u; + //prologue + 8GB fault buffer + 4096 addtional buffer for safety + static constexpr uint64_t total_memory_per_slice = memory_prologue_size + UINT64_C(0x200000000) + UINT64_C(4096); public: - memory(); + explicit memory(uint64_t max_pages); ~memory(); + memory(const memory&) = delete; + memory& operator=(const memory&) = delete; + void reset(uint64_t max_pages); uint8_t* const zero_page_memory_base() const { return zeropage_base; } uint8_t* const full_page_memory_base() const { return fullpage_base; } @@ -47,6 +47,8 @@ class memory { static constexpr uintptr_t linear_memory = 0; static constexpr uintptr_t cb_offset = wcb_allowance + mutable_global_size + table_size; static constexpr uintptr_t first_intrinsic_offset = cb_offset + 8u; + // The maximum amount of data that PIC code can include in the prologue + static constexpr uintptr_t max_prologue_size = mutable_global_size + table_size; static_assert(-cb_offset == EOS_VM_OC_CONTROL_BLOCK_OFFSET, "EOS VM OC control block offset has slid out of place somehow"); static_assert(stride == EOS_VM_OC_MEMORY_STRIDE, "EOS VM OC memory stride has slid out of place somehow"); @@ -62,4 +64,4 @@ class memory { }}} #define OFFSET_OF_CONTROL_BLOCK_MEMBER(M) (-(int)eosio::chain::eosvmoc::memory::cb_offset + (int)offsetof(eosio::chain::eosvmoc::control_block, M)) -#define OFFSET_OF_FIRST_INTRINSIC ((int)-eosio::chain::eosvmoc::memory::first_intrinsic_offset) \ No newline at end of file +#define OFFSET_OF_FIRST_INTRINSIC ((int)-eosio::chain::eosvmoc::memory::first_intrinsic_offset) diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/stack.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/stack.hpp new file mode 100644 index 00000000000..9eac158d0f7 --- /dev/null +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/stack.hpp @@ -0,0 +1,42 @@ +#pragma once + +#include + +namespace eosio { namespace chain { namespace eosvmoc { + +extern "C" void eosvmoc_switch_stack(void* stack, void(*fn)(void*), void* data); + +// Allows wasm code to run with a stack whose size can be adjusted based +// on the configurable max_call_depth. It is assumed that max_call_depth +// is rarely changed. +struct execution_stack { + // Must match the limit enforced by codegen. + static constexpr std::size_t max_bytes_per_frame = 16*1024; + + void * stack_top = nullptr; + std::size_t stack_size = 0; + // Assume that the default stack is large enough if we can't consume more than 4 MB + std::size_t call_depth_limit = 4*1024*1024 / max_bytes_per_frame; + + execution_stack() = default; + execution_stack(const execution_stack&) = delete; + execution_stack& operator=(const execution_stack&) = delete; + + void reset(std::size_t max_call_depth); + void reset(); + + template + void run(F&& f) { + if (stack_top) { + eosvmoc_switch_stack(stack_top, [](void* data) { (*static_cast(data))(); }, &f); + } else { + f(); + } + } + + ~execution_stack() { + reset(); + } +}; + +}}} diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm.hpp index 5119a5c386f..6f4bf2ceccd 100644 --- a/libraries/chain/include/eosio/chain/webassembly/eos-vm.hpp +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm.hpp @@ -1,119 +1,38 @@ #pragma once -#if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED) - #include #include #include #include +#include +#include #include //eos-vm includes #include -// eosio specific specializations -namespace eosio { namespace vm { - - template<> - struct wasm_type_converter { - static auto from_wasm(uint64_t val) { - return eosio::chain::name{val}; - } - static auto to_wasm(eosio::chain::name val) { - return val.to_uint64_t(); - } - }; - - template - struct wasm_type_converter : linear_memory_access { - auto from_wasm(void* val) { - validate_ptr(val, 1); - return eosio::vm::aligned_ptr_wrapper{val}; - } - }; - - template<> - struct wasm_type_converter : linear_memory_access { - void* to_wasm(char* val) { - validate_ptr(val, 1); - return val; - } - }; - - template - struct wasm_type_converter : linear_memory_access { - auto from_wasm(uint32_t val) { - EOS_VM_ASSERT( val != 0, wasm_memory_exception, "references cannot be created for null pointers" ); - void* ptr = get_ptr(val); - validate_ptr(ptr, 1); - return eosio::vm::aligned_ref_wrapper{ptr}; - } - }; - - template - struct wasm_type_converter> : linear_memory_access { - auto from_wasm(void* ptr, uint32_t size) { - validate_ptr(ptr, size); - return aligned_array_wrapper(ptr, size); - } - }; - - template<> - struct wasm_type_converter> : linear_memory_access { - auto from_wasm(void* ptr, uint32_t size) { - validate_ptr(ptr, size); - return eosio::chain::array_ptr((char*)ptr); - } - // memcpy/memmove - auto from_wasm(void* ptr, eosio::chain::array_ptr /*src*/, uint32_t size) { - validate_ptr(ptr, size); - return eosio::chain::array_ptr((char*)ptr); - } - // memset - auto from_wasm(void* ptr, int /*val*/, uint32_t size) { - validate_ptr(ptr, size); - return eosio::chain::array_ptr((char*)ptr); - } - }; - - template<> - struct wasm_type_converter> : linear_memory_access { - auto from_wasm(void* ptr, uint32_t size) { - validate_ptr(ptr, size); - return eosio::chain::array_ptr((char*)ptr); - } - // memcmp - auto from_wasm(void* ptr, eosio::chain::array_ptr /*src*/, uint32_t size) { - validate_ptr(ptr, size); - return eosio::chain::array_ptr((char*)ptr); - } - }; - - template - struct construct_derived { - static auto &value(Ctx& ctx) { return ctx.trx_context; } - }; - - template <> - struct construct_derived { - static auto &value(eosio::chain::apply_context& ctx) { return ctx; } - }; - - template<> - struct wasm_type_converter : linear_memory_access { - auto from_wasm(void* ptr) { - validate_c_str(ptr); - return eosio::chain::null_terminated_ptr{ static_cast(ptr) }; - } - }; - -}} // ns eosio::vm - namespace eosio { namespace chain { namespace webassembly { namespace eos_vm_runtime { +struct apply_options; + +}} + +template +using eos_vm_backend_t = eosio::vm::backend; + +template +using eos_vm_null_backend_t = eosio::vm::backend; + +namespace webassembly { namespace eos_vm_runtime { + using namespace fc; using namespace eosio::vm; -using namespace eosio::chain::webassembly::common; + +void validate(const bytes& code, const whitelisted_intrinsics_type& intrinsics ); + +void validate(const bytes& code, const wasm_config& cfg, const whitelisted_intrinsics_type& intrinsics ); + +struct apply_options; template class eos_vm_runtime : public eosio::chain::wasm_runtime_interface { @@ -129,22 +48,10 @@ class eos_vm_runtime : public eosio::chain::wasm_runtime_interface { // todo: managing this will get more complicated with sync calls; // immediately_exit_currently_running_module() should probably // move from wasm_runtime_interface to wasm_instantiated_module_interface. - backend* _bkend = nullptr; // non owning pointer to allow for immediate exit + eos_vm_backend_t* _bkend = nullptr; // non owning pointer to allow for immediate exit template friend class eos_vm_instantiated_module; }; -} } } }// eosio::chain::webassembly::wabt_runtime - -#define __EOS_VM_INTRINSIC_NAME(LBL, SUF) LBL##SUF -#define _EOS_VM_INTRINSIC_NAME(LBL, SUF) __INTRINSIC_NAME(LBL, SUF) - -#define _REGISTER_EOS_VM_INTRINSIC(CLS, MOD, METHOD, WASM_SIG, NAME, SIG) \ - eosio::vm::registered_function _EOS_VM_INTRINSIC_NAME(__eos_vm_intrinsic_fn, __COUNTER__)(std::string(MOD), std::string(NAME)); - -#else - -#define _REGISTER_EOS_VM_INTRINSIC(CLS, MOD, METHOD, WASM_SIG, NAME, SIG) - -#endif +}}}}// eosio::chain::webassembly::eos_vm_runtime diff --git a/libraries/chain/include/eosio/chain/webassembly/interface.hpp b/libraries/chain/include/eosio/chain/webassembly/interface.hpp new file mode 100644 index 00000000000..84787d7c085 --- /dev/null +++ b/libraries/chain/include/eosio/chain/webassembly/interface.hpp @@ -0,0 +1,1966 @@ +#pragma once + +#include +#include +#include +#include + +namespace eosio { namespace chain { +class apply_context; +namespace webassembly { + + class interface { + public: + interface(apply_context& ctx) : context(ctx) {} + + inline apply_context& get_context() { return context; } + inline const apply_context& get_context() const { return context; } + + /** + * Retrieve the signed_transaction.context_free_data[index]. + * + * @ingroup context-free + * @param index - the index of the context_free_data entry to retrieve. + * @param[out] buffer - output buffer of the context_free_data entry. + * + * @retval -1 if the index is not valid. + * @retval size of the cfd if the buffer is empty, otherwise return the amount of data copied onto the buffer. + */ + int32_t get_context_free_data(uint32_t index, legacy_span buffer) const; + + /** + * Check if a feature is found on the activation set. + * + * @ingroup privileged + * @param feature_name - 256-bit digest representing the feature to query. + * + * @return false (deprecated) + * + * @deprecated + */ + int32_t is_feature_active(int64_t feature_name) const; + + /** + * Activate a a consensus protocol upgrade. + * + * @ingroup privileged + * @param feature_name - 256-bit digest representing the feature to activate. + * + * @deprecated + */ + void activate_feature(int64_t feature_name) const; + + /** + * Allows a privileged smart contract, e.g. the system contract, to pre-activate a consensus protocol upgrade feature. + * + * @ingroup privileged + * @param feature_digest - 256-bit digest representing the feature to pre-activate. + */ + void preactivate_feature(legacy_ptr feature_digest); + + /** + * Set the resource limits of an account. + * + * @ingroup privileged + * + * @param account - name of the account whose resource limit to be set. + * @param ram_bytes - ram limit in absolute bytes. + * @param net_weight - fractionally proportionate net limit of available resources based on (weight / total_weight_of_all_accounts). + * @param cpu_weight - fractionally proportionate cpu limit of available resources based on (weight / total_weight_of_all_accounts). + */ + void set_resource_limits(account_name account, int64_t ram_bytes, int64_t net_weight, int64_t cpu_weight); + + /** + * Get the resource limits of an account + * + * @ingroup privileged + * + * @param account - name of the account whose resource limit to get. + * @param[out] ram_bytes - output to hold retrieved ram limit in absolute bytes. + * @param[out] net_weight - output to hold net weight. + * @param[out] cpu_weight - output to hold cpu weight. + */ + void get_resource_limits(account_name account, legacy_ptr ram_bytes, legacy_ptr net_weight, legacy_ptr cpu_weight) const; + + /** + * Get the current wasm limits configuration. + * + * The structure of the parameters is as follows: + * + * - max_mutable_global_bytes + * The maximum total size (in bytes) used for mutable globals. + * i32 and f32 consume 4 bytes and i64 and f64 consume 8 bytes. + * Const globals are not included in this count. + * + * - max_table_elements + * The maximum number of elements of a table. + * + * - max_section_elements + * The maximum number of elements in each section. + * + * - max_linear_memory_init + * The size (in bytes) of the range of memory that may be initialized. + * Data segments may use the range [0, max_linear_memory_init). + * + * - max_func_local_bytes + * The maximum total size (in bytes) used by parameters and local variables in a function. + * + * - max_nested_structures + * The maximum nesting depth of structured control instructions. + * The function itself is included in this count. + * + * - max_symbol_bytes + * The maximum size (in bytes) of names used for import and export. + * + * - max_module_bytes + * The maximum total size (in bytes) of a wasm module. + * + * - max_code_bytes + * The maximum size (in bytes) of each function body. + * + * - max_pages + * The maximum number of 64 KiB pages of linear memory that a contract can use. + * Enforced when an action is executed. The initial size of linear memory is also checked at setcode. + * + * - max_call_depth + * The maximum number of functions that may be on the stack. Enforced when an action is executed. + * + * @ingroup privileged + * + * @param[out] packed_parameters the ouput for the parameters. + * @param max_version has no effect, but should be 0. + * + * @return the size of the packed parameters if packed_parameters is empty, otherwise it returns the amount of data written in packed_parameters. + */ + uint32_t get_wasm_parameters_packed( span packed_parameters, uint32_t max_version ) const; + + /** + * Set the configuration for wasm limits. + * + * See get_wasm_parameters_packed documentation for more details on the structure of the packed_parameters. + * + * @ingroup privileged + * + * @param packed_parameters - a span containing the packed configuration to set. + */ + void set_wasm_parameters_packed( span packed_parameters ); + + /** + * Update a single resource limit associated with an account. + * + * @ingroup privileged + * + * @param account - the account whose limits are being modified. + * @param resource - the resource to update, which should be either ram, cpu, or net. + * @param limit - the new limit. A value of -1 means unlimited. + * + * @pre limit >= -1 + */ + void set_resource_limit(account_name account, name resource, int64_t limit); + + /** + * Get a single resource limit associated with an account. + * + * @ingroup privileged + * + * @param account - the account whose limits are being modified + * @param resource - the name of the resource limit which should be either ram, cpu, or net. + * + * @return the limit on the resource requested. + */ + int64_t get_resource_limit(account_name account, name resource) const; + + /** + * Proposes a schedule change using the legacy producer key format. + * + * @ingroup privileged + * + * @param packed_producer_schedule - vector of producer keys + * + * @return -1 if proposing a new producer schedule was unsuccessful, otherwise returns the version of the new proposed schedule. + */ + int64_t set_proposed_producers(legacy_span packed_producer_schedule); + + /** + * Proposes a schedule change with extended features. + * + * Valid formats: + * 0 : serialized array of producer_keys. Using this format is exactly equivalent to set_proposed_producers. + * 1 : serialized array of producer_authority's. + * + * @ingroup privileged + * + * @param packed_producer_format - format of the producer data blob. + * @param packed_producer_schedule - packed data of representing the producer schedule in the format indicated. + * + * @return -1 if proposing a new producer schedule was unsuccessful, otherwise returns the version of the new proposed schedule. + */ + int64_t set_proposed_producers_ex(uint64_t packed_producer_format, legacy_span packed_producer_schedule); + + /** + * Retrieve the blockchain config parameters. + * + * @ingroup privileged + * + * @param[out] packed_blockchain_parameters - output buffer of the blockchain parameters. + * + * return the number of bytes copied to the buffer, or number of bytes required if the buffer is empty. + */ + uint32_t get_blockchain_parameters_packed(legacy_span packed_blockchain_parameters) const; + + /** + * Set the blockchain parameters. + * + * @ingroup privileged + * + * @param packed_blockchain_parameters - a span containing the packed blockchain config parameters. + */ + void set_blockchain_parameters_packed(legacy_span packed_blockchain_parameters); + + /** + * Retrieve the blockchain config parameters. + * The input buffer is a packed data stream which represents an encoded sequence of parameter_id pairs with the following format: + * |varuint32:sequence_length | varuint32:parameter_id | ... + * The output buffer is a packed data stream which represents an encoded sequence of parameter_id:paramter_value pairs with the following format: + * |varuint32:sequence_length | varuint32:parameter_id | :parameter_value | ... + * The encoding of parameter_values should be specific to the parameter being set + * The output buffer format should be valid input for set_parameters_packed. + * For each known parameter_id in the input sequence there should be an associated entry in the output sequence with the current encoded parameter_value. + * + * @brief Retrieve the blockchain config parameters. + * @ingroup privileged + * + * @param packed_parameter_ids - the input buffer with the format as described above. + * @param[out] packed_parameters - the output buffer with the format as described above. + */ + uint32_t get_parameters_packed( span packed_parameter_ids, span packed_parameters) const; + + /** + * Set the blockchain parameters. + * It allows a system contract the ability to set parameters in a flexible manner. + * The input buffer is a packed data stream which represents an encoded sequence of parameter_id:paramter_value pairs with the following format: + * |varuint32:sequence_length | varuint32:parameter_id | :parameter_value | ... + * The encoding of parameter_values should be specific to the parameter being set. + * Having duplicate parameter_ids encoded in the sequence should result in aborting the transaction context. + * The presence of a parameter_id which is unknown OR which is known but tied to an unactivated consensus protocol + * should result in aborting the transaction context. + * There are no requirement for the ordering of items in the sequence. + * + * @brief Set the blockchain parameters in a flexible manner. + * @ingroup privileged + * + * @param packed_parameters - buffer to hold the packed data with the format described above. + */ + void set_parameters_packed( span packed_parameters ); + + /** + * Gets the maximum key size, maximum value size, and maximum iterators of a kv database and returns the size of the data. + * The kv parameters are encoded as 16 bytes, representing four 32-bit little-endian values. + * ``` + * +-------+---------------+---------------+---------------+---------------+ + * | byte | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 |10 |11 |12 |13 |14 |15 | + * +-------+---------------+---------------+---------------+---------------+ + * | field | version | key limit | value limit | max iterators | + * +-------+---------------+---------------+---------------+---------------+ + * | type | 0 | 32-bits LE | 32-bits LE | 32-bits LE | + * +-------+---------------+---------------+---------------+---------------+ + * ``` + * + * @ingroup privileged + * @param[out] packed_kv_parameters - the buffer containing the packed kv parameters. + * @param max_version - has no effect, but should be 0. + * + * @return Returns the size required in the buffer (if the buffer is too small, nothing is written). + * + */ + uint32_t get_kv_parameters_packed(span packed_kv_parameters, uint32_t max_version) const; + + /** + * Sets the maximum key size, and maximum value size, and maximum iterators of a kv database. + * Each database has independent limits. The key and value limits only apply to new items. + * They do not apply to items written before they were applied. + * If the database is invalid, if version is non-zero, or if buffer_size is less than 16, aborts the transaction. + * + * @ingroup privileged + * @param packed_kv_parameters - the buffer containing the packed kv parameters to be set. + */ + void set_kv_parameters_packed(span packed_kv_parameters); + + /** + * Check if an account is privileged. + * + * @ingroup privileged + * @param account - name of the account to be checked. + * + * @retval true if the account is privileged + * @retval false otherwise + */ + bool is_privileged(account_name account) const; + + /** + * Set the privileged status of an account. + * + * @ingroup privileged + * @param account - name of the account that we want to give the privileged status. + * @param is_priv - privileged status (true or false). + */ + void set_privileged(account_name account, bool is_priv); + + // softfloat api + float _eosio_f32_add(float, float) const; + float _eosio_f32_sub(float, float) const; + float _eosio_f32_div(float, float) const; + float _eosio_f32_mul(float, float) const; + float _eosio_f32_min(float, float) const; + float _eosio_f32_max(float, float) const; + float _eosio_f32_copysign(float, float) const; + float _eosio_f32_abs(float) const; + float _eosio_f32_neg(float) const; + float _eosio_f32_sqrt(float) const; + float _eosio_f32_ceil(float) const; + float _eosio_f32_floor(float) const; + float _eosio_f32_trunc(float) const; + float _eosio_f32_nearest(float) const; + bool _eosio_f32_eq(float, float) const; + bool _eosio_f32_ne(float, float) const; + bool _eosio_f32_lt(float, float) const; + bool _eosio_f32_le(float, float) const; + bool _eosio_f32_gt(float, float) const; + bool _eosio_f32_ge(float, float) const; + double _eosio_f64_add(double, double) const; + double _eosio_f64_sub(double, double) const; + double _eosio_f64_div(double, double) const; + double _eosio_f64_mul(double, double) const; + double _eosio_f64_min(double, double) const; + double _eosio_f64_max(double, double) const; + double _eosio_f64_copysign(double, double) const; + double _eosio_f64_abs(double) const; + double _eosio_f64_neg(double) const; + double _eosio_f64_sqrt(double) const; + double _eosio_f64_ceil(double) const; + double _eosio_f64_floor(double) const; + double _eosio_f64_trunc(double) const; + double _eosio_f64_nearest(double) const; + bool _eosio_f64_eq(double, double) const; + bool _eosio_f64_ne(double, double) const; + bool _eosio_f64_lt(double, double) const; + bool _eosio_f64_le(double, double) const; + bool _eosio_f64_gt(double, double) const; + bool _eosio_f64_ge(double, double) const; + double _eosio_f32_promote(float) const; + float _eosio_f64_demote(double) const; + int32_t _eosio_f32_trunc_i32s(float) const; + int32_t _eosio_f64_trunc_i32s(double) const; + uint32_t _eosio_f32_trunc_i32u(float) const; + uint32_t _eosio_f64_trunc_i32u(double) const; + int64_t _eosio_f32_trunc_i64s(float) const; + int64_t _eosio_f64_trunc_i64s(double) const; + uint64_t _eosio_f32_trunc_i64u(float) const; + uint64_t _eosio_f64_trunc_i64u(double) const; + float _eosio_i32_to_f32(int32_t) const; + float _eosio_i64_to_f32(int64_t) const; + float _eosio_ui32_to_f32(uint32_t) const; + float _eosio_ui64_to_f32(uint64_t) const; + double _eosio_i32_to_f64(int32_t) const; + double _eosio_i64_to_f64(int64_t) const; + double _eosio_ui32_to_f64(uint32_t) const; + double _eosio_ui64_to_f64(uint64_t) const; + + /** + * Get the list of active producer names. + * + * @ingroup producer + * @param[out] producers - output buffer containing the names of the current active producer names. + * + * @return number of bytes required (if the buffer is empty), or the number of bytes written to the buffer. + */ + int32_t get_active_producers(legacy_span producers) const; + + /** + * Tests a given public key with the recovered public key from digest and signature. + * + * @ingroup crypto + * @param digest - digest of the message that was signed. + * @param sig - signature. + * @param pub - public key. + */ + void assert_recover_key(legacy_ptr digest, legacy_span sig, legacy_span pub) const; + + /** + * Calculates the public key used for a given signature on a given digest. + * + * @ingroup crypto + * @param digest - digest of the message that was signed. + * @param sig - signature. + * @param[out] pub - output buffer for the public key result. + * + * @return size of data written on the buffer. + */ + int32_t recover_key(legacy_ptr digest, legacy_span sig, legacy_span pub) const; + + /** + * Tests if the sha256 hash generated from data matches the provided digest. + * + * @ingroup crypto + * @param data - a span containing the data you want to hash. + * @param hash_val - digest to compare to. + */ + void assert_sha256(legacy_span data, legacy_ptr hash_val) const; + + /** + * Tests if the sha1 hash generated from data matches the provided digest. + * + * @ingroup crypto + * @param data - a span containing the data you want to hash. + * @param hash_val - digest to compare to. + */ + void assert_sha1(legacy_span data, legacy_ptr hash_val) const; + + /** + * Tests if the sha512 hash generated from data matches the provided digest. + * + * @ingroup crypto + * @param data - a span containing the data you want to hash. + * @param hash_val - digest to compare to. + */ + void assert_sha512(legacy_span data, legacy_ptr hash_val) const; + + /** + * Tests if the ripemd160 hash generated from data matches the provided digest. + * + * @ingroup crypto + * @param data - a span containing the data you want to hash. + * @param hash_val - digest to compare to. + */ + void assert_ripemd160(legacy_span data, legacy_ptr hash_val) const; + + /** + * Hashes data using SHA256. + * + * @ingroup crypto + * @param data - a span containing the data. + * @param[out] hash_val - the resulting digest. + */ + void sha256(legacy_span data, legacy_ptr hash_val) const; + + /** + * Hashes data using SHA1. + * + * @ingroup crypto + * @param data - a span containing the data. + * @param[out] hash_val - the resulting digest. + */ + void sha1(legacy_span data, legacy_ptr hash_val) const; + + /** + * Hashes data using SHA512. + * + * @ingroup crypto + * @param data - a span containing the data. + * @param[out] hash_val - the hash + */ + void sha512(legacy_span data, legacy_ptr hash_val) const; + + /** + * Hashes data using RIPEMD160. + * + * @ingroup crypto + * @param data - a span containing the data. + * @param[out] hash_val - computed digest. + */ + void ripemd160(legacy_span data, legacy_ptr hash_val) const; + + /** + * Checks if a transaction is authorized by a provided set of keys and permissions. + * + * @ingroup permission + * @param trx_data - serialized transaction. + * @param pubkeys_data - serialized vector of provided public keys. + * @param perms_data - serialized vector of provided permissions (empty permission name acts as wildcard). + * + * @retval true if transaction is authorized. + * @retval false otherwise. + */ + bool check_transaction_authorization(legacy_span trx_data, legacy_span pubkeys_data, legacy_span perms_data) const; + + /** + * Checks if a permission is authorized by a provided delay and a provided set of keys and permissions. + * + * @ingroup permission + * @param account - the account owner of the permission. + * @param permission - the name of the permission to check for authorization. + * @param pubkeys_data - serialized vector of provided public keys. + * @param perms_data - serialized vector of provided permissions (empty permission name acts as wildcard). + * @param delay_us - the provided delay in microseconds (cannot exceed INT64_MAX) + * + * @retval true if permission is authorized. + * @retval false otherwise. + */ + bool check_permission_authorization(account_name account, permission_name permission, legacy_span pubkeys_data, legacy_span perms_data, uint64_t delay_us) const; + + /** + * Returns the last used time of a permission. + * + * @ingroup permission + * @param account - the account owner of the permission. + * @param permission - the name of the permission. + * + * @return the last used time (in microseconds since Unix epoch) of the permission. + */ + int64_t get_permission_last_used(account_name account, permission_name permission) const; + + /** + * Returns the creation time of an account. + * + * @ingroup permission + * @param account - the account name. + * + * @return the creation time (in microseconds since Unix epoch) of the account. + */ + int64_t get_account_creation_time(account_name account) const; + + /** + * Verifies that an account exists in the set of provided auths on an action. Fails if not found. + * + * @ingroup authorization + * @param account - the name of the account to be verified. + */ + void require_auth(account_name account) const; + + /** + * Verifies that an account with a specific permission exists in the set of provided auths on an action, + * + * @ingroup authorization + * @param account - the name of the account to be verified. + * @param permission - the name of the permission to be verified. + */ + void require_auth2(account_name account, permission_name permission) const; + + /** + * Test whether an account exists in the set of provided auths on an action. + * + * @ingroup authorization + * @param account - name of the account to be tested. + * + * @retval true if the action has an auth with the account name. + * @retval false otherwise. + */ + bool has_auth(account_name account) const; + + /** + * Add the specified account to set of accounts to be notified. + * + * @ingroup authorization + * @param recipient - account to be notified. + */ + void require_recipient(account_name recipient); + + /** + * Verifies that n is an existing account. + * + * @ingroup authorization + * @param account - name of the account to check. + * + * @return true if the account exists. + * @return false otherwise. + */ + bool is_account(account_name account) const; + + /** + * Returns the time in microseconds from 1970 of the current block. + * + * @ingroup system + * + * @return time in microseconds from 1970 of the current block. + */ + uint64_t current_time() const; + + /** + * Returns the transaction's publication time. + * + * @ingroup system + * + * @return time in microseconds from 1970 of the publication_time. + */ + uint64_t publication_time() const; + + /** + * Check if specified protocol feature has been activated. + * + * @ingroup system + * @param feature_digest - digest of the protocol feature. + * + * @retval true if the specified protocol feature has been activated. + * @retval false otherwise. + */ + bool is_feature_activated(legacy_ptr feature_digest) const; + + /** + * Return the name of the account that sent the current inline action. + * + * @ingroup system + * @return name of account that sent the current inline action (empty name if not called from inline action). + */ + name get_sender() const; + + /** + * Aborts processing of this action and unwinds all pending changes. + * + * @ingroup context-free + */ + void abort() const; + + /** + * Aborts processing of this action if the test condition is false. + * + * @ingroup context-free + * @param condition - test condition. + * @param msg - string explaining the reason for failure. + */ + void eosio_assert(bool condition, null_terminated_ptr msg) const; + + /** + * Aborts processing of this action if the test condition is false. + * + * @ingroup context-free + * @param condition - test condition. + * @param msg - string explaining the reason for failure. + */ + void eosio_assert_message(bool condition, legacy_span msg) const; + + /** + * Aborts processing of this action if the test condition is false. + * It can be used to provide an error code rather than a message string on assertion checks. + * If the assertion fails, the provided error code will be made available through the exception message. + * + * @ingroup context-free + * @param condition - test condition. + * @param error_code - the error code associated. + */ + void eosio_assert_code(bool condition, uint64_t error_code) const; + + /** + * This method will abort execution of wasm without failing the contract. + * + * @ingroup context-free + * @param code - the exit code + */ + void eosio_exit(int32_t code) const; + + /** + * Copy up to length bytes of the current action data to the specified location. + * + * @ingroup action + * @param memory - a pointer where up to length bytes of the current action data will be copied. + * + * @return the number of bytes copied to msg, or number of bytes that can be copied if an empty span is passed. + */ + int32_t read_action_data(legacy_span memory) const; + + /** + * Get the length of the current action's data field. This method is useful for dynamically sized actions. + * + * @ingroup action + * @return the length of the current action's data field + */ + int32_t action_data_size() const; + + /** + * Get the current receiver of the action. + * + * @ingroup action + * @return the name of the receiver + */ + name current_receiver() const; + + /** + * Sets a value (packed blob char array) to be included in the action receipt. + * + * @ingroup action + * @param packed_blob - the packed blob + */ + void set_action_return_value(span packed_blob); + + /** + * Print a string. + * + * @ingroup console + * @param str - the string to print + */ + void prints(null_terminated_ptr str); + + /** + * Prints string up to given length. + * + * @ingroup console + * @param str - the string to print. + */ + void prints_l(legacy_span str); + + /** + * Prints value as a 64 bit signed integer. + * + * @ingroup console + * @param val - 64 bit signed integer to be printed. + */ + void printi(int64_t val); + + /** + * Prints value as a 64 bit unsigned integer. + * + * @ingroup console + * @param val - 64 bit unsigned integer to be printed. + */ + void printui(uint64_t val); + + /** + * Prints value as a 128 bit signed integer. + * + * @ingroup console + * @param val - 128 bit signed integer to be printed. + */ + void printi128(legacy_ptr val); + + /** + * Prints value as a 128 bit unsigned integer. + * + * @ingroup console + * @param val - 128 bit unsigned integer to be printed. + */ + void printui128(legacy_ptr val); + + /** + * Prints value as single-precision floating point number. + * + * @ingroup console + * @param val - single-precision floating point number to be printed. + */ + void printsf(float32_t val); + + /** + * Prints value as double-precision floating point number. + * + * @ingroup console + * @param val - double-precision floating point number to be printed + */ + void printdf(float64_t val); + + /** + * Prints value as quadruple-precision floating point number. + * + * @ingroup console + * @param val - a pointer to the quadruple-precision floating point number to be printed + */ + void printqf(legacy_ptr val); + + /** + * Prints a 64 bit names as base32 encoded string. + * + * @ingroup console + * @param value - 64 bit name to be printed + */ + void printn(name value); + + /** + * Prints a 64 bit names as base32 encoded string + * + * @ingroup console + * @param data - Hex name to be printed. + */ + void printhex(legacy_span data); + + /** + * Store a record in a primary 64-bit integer index table. + * + * @ingroup database primary-index + * + * @param scope - the scope where the table resides (implied to be within the code of the current receiver). + * @param table - the name of the table within the current scope context. + * @param payer - the account that pays for the storage. + * @param id - id of the entry. + * @param buffer - record to store. + * + * @return iterator to the newly created table row. + * @post a new entry is created in the table. + */ + int32_t db_store_i64(uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, legacy_span buffer); + + /** + * Update a record in a primary 64-bit integer index table. + * + * @ingroup database primary-index + * @param itr - iterator to the table row containing the record to update. + * @param payer - the account that pays for the storage costs. + * @param buffer - new updated record. + * + * @remark This function does not allow changing the primary key of a + * table row. The serialized data that is stored in the table row of a + * primary table may include a primary key and that primary key value + * could be changed by the contract calling the db_update_i64 intrinsic; + * but that does not change the actual primary key of the table row. + * + * @pre `itr` points to an existing table row in the table. + * @post the record contained in the table row pointed to by `itr` is replaced with the new updated record. + */ + void db_update_i64(int32_t itr, uint64_t payer, legacy_span buffer); + + /** + * Remove a record inside a primary 64-bit integer index table. + * + * @ingroup database primary-index + * @param itr - the iterator to the table row to remove. + * + * @pre `itr` points to an existing table row in the tab. + */ + void db_remove_i64(int32_t itr); + + /** + * Get a record in a primary 64-bit integer index table. + * + * @ingroup database primary-index + * @param itr - the iterator to the table row containing the record to retrieve. + * @param[out] buffer - the buffer which will be filled with the retrieved record. + * + * @return size of the data copied into the buffer if buffer is not empty, or size of the retrieved record if the buffer is empty. + * @pre `itr` points to an existing table row in the table. + * @post `buffer` will be filled with the retrieved record (truncated to the first `len` bytes if necessary). + */ + int32_t db_get_i64(int32_t itr, legacy_span buffer); + + /** + * Find the table row following the referenced table row in a primary 64-bit integer index table. + * + * @ingroup database primary-index + * @param itr - the iterator to the referenced table row. + * @param[out] primary - pointer to a `uint64_t` variable which will have its value set to the primary key of the next table row. + * + * @return iterator to the table row following the referenced table row (or the end iterator of the table if the referenced table row is the last one in the table). + * + * @post '*primary' will be replaced with the primary key of the table row following the referenced table row if it exists, otherwise primary will be left untouched. + */ + int32_t db_next_i64(int32_t itr, legacy_ptr primary); + + /** + * Find the table row preceding the referenced table row in a primary 64-bit integer index table. + * + * @ingroup database primary-index + * @param itr - the iterator to the referenced table row. + * @param[out] primary - pointer to a `uint64_t` variable which will have its value set to the primary key of the next table row. + * + * @return iterator to the table row preceding the referenced table row assuming one exists (it will return -1 if the referenced table row is the first one in the table). + * @post '*primary' will be replaced with the primary key of the table row preceding the referenced table row if it exists, otherwise primary will be left untouched. + */ + int32_t db_previous_i64(int32_t itr, legacy_ptr primary); + + /** + * Find a table row in a primary 64-bit integer index table by primary key. + * + * @ingroup database primary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * @param id - the primary key of the record to look up. + * + * @return iterator to the table row with a primary key equal to id or the end iterator of the table if the table row could not be found. + */ + int32_t db_find_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id); + + /** + * Find the table row in a primary 64-bit integer index table that matches the lowerbound condition for a given primary key. + * Lowerbound record is the first nearest record which primary key is <= the given key. + * + * @ingroup database primary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * @param id - the primary key used as a pivot to determine the lowerbound record. + * + * @return iterator to the lowerbound record or the end iterator of the table if the table row could not be found. + */ + int32_t db_lowerbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id); + + /** + * Find the table row in a primary 64-bit integer index table that matches the upperbound condition for a given primary key. + * The table row that matches the upperbound condition is the first table row in the table with the lowest primary key that is > the given key. + * + * @ingroup database primary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * @param id - the primary key used as a pivot to determine the upperbound record. + * + * @return iterator to the upperbound record or the end iterator of the table if the table row could not be found. + */ + int32_t db_upperbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id); + + /** + * Get an iterator representing just-past-the-end of the last table row of a primary 64-bit integer index table. + * + * @ingroup database primary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * + * @return end iterator of the table. + */ + int32_t db_end_i64(uint64_t code, uint64_t scope, uint64_t table); + + /** + * Store an association of a 64-bit integer secondary key to a primary key in a secondary 64-bit integer index table. + * + * @ingroup database uint64_t-secondary-index + * @param scope - the scope where the table resides (implied to be within the code of the current receiver). + * @param table - the table name. + * @param payer - the account that is paying for this storage. + * @param id - the primary key to which to associate the secondary key. + * @param secondary - the pointer to the key of the secondary index to store. + * + * @return iterator to the newly created secondary index. + * @post new secondary key association between primary key `id` and secondary key `*secondary` is created in the secondary 64-bit integer index table. + */ + int32_t db_idx64_store(uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, legacy_ptr secondary); + + /** + * Update an association for a 64-bit integer secondary key to a primary key in a secondary 64-bit integer index table. + * + * @ingroup database uint64_t-secondary-index + * @param iterator - the iterator to the table row containing the secondary key association to update. + * @param payer - the account that pays for the storage costs. + * @param secondary - pointer to the **new** secondary key that will replace the existing one of the association. + * + * @pre `iterator` points to an existing table row in the table. + * @post the secondary key of the table row pointed to by `iterator` is replaced by `*secondary`. + */ + void db_idx64_update(int32_t iterator, uint64_t payer, legacy_ptr secondary); + + /** + * Remove a table row from a secondary 64-bit integer index table. + * + * @ingroup database uint64_t-secondary-index + * @param iterator - iterator to the table row to remove. + * + * @pre `iterator` points to an existing table row in the table. + * @post the table row pointed to by `iterator` is removed and the associated storage costs are refunded to the payer. + */ + void db_idx64_remove(int32_t iterator); + + /** + * Find a table row in a secondary 64-bit integer index table by secondary key. + * + * @ingroup database uint64_t-secondary-index + * + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * @param secondary - the pointer to the secondary index key. + * @param[out] primary - pointer to a 'uint64_t' variable which will have its value set to the primary key of the found table row. + * + * @return iterator to the first table row with a secondary key equal to `*secondary` or the end iterator of the table if the table row could not be found. + * @post If and only if the table row is found, `*primary` will be replaced with the primary key of the found table row. + */ + int32_t db_idx64_find_secondary(uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary); + + /** + * Find a table row in a secondary 64-bit integer index table by primary key. + * + * @ingroup database uint64_t-secondary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * @param[out] secondary - pointer to a 'uint64_t' variable which will have its value set to the secondary key of the found table row. + * @param primary - the primary key of the table row to look up. + * + * @return iterator to the table row with a primary key equal to `primary` or the end iterator of the table if the table row could not be found. + * @post If and only if the table row is found, `*secondary` will be replaced with the secondary key of the found table row. + */ + int32_t db_idx64_find_primary(uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, uint64_t primary); + + /** + * Find the table row in a secondary 64-bit integer index table that matches the lowerbound condition for a given secondary key. + * Lowerbound secondary index is the first secondary index which key is <= the given secondary index key. + * + * @ingroup database uint64_t-secondary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * @param[out] secondary - pointer to secondary key first used to determine the lowerbound and which is then replaced with the secondary key of the found table row. + * @param[out] primary - pointer to a `uint64_t` variable which will have its value set to the primary key of the found table row. + * + * @return iterator to the found table row or the end iterator of the table if the table row could not be found. + * + * @post If and only if the table row is found, `*secondary` will be replaced with the secondary key of the found table row. + * @post If and only if the table row is found, `*primary` will be replaced with the primary key of the found table row. + */ + int32_t db_idx64_lowerbound(uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary); + + /** + * Find the table row in a secondary 64-bit integer index table that matches the upperbound condition for a given secondary key. + * The table row that matches the upperbound condition is the first table row in the table with the lowest secondary key that is > the given key. + * + * @ingroup database uint64_t-secondary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * @param[out] secondary - pointer to secondary key first used to determine the upperbound and which is then replaced with the secondary key of the found table row. + * @param[out] primary - pointer to a `uint64_t` variable which will have its value set to the primary key of the found table row. + * + * @return iterator to the found table row or the end iterator of the table if the table row could not be found. + */ + int32_t db_idx64_upperbound(uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary); + + /** + * Get an end iterator representing just-past-the-end of the last table row of a secondary 64-bit integer index table. + * + * @ingroup database uint64_t-secondary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * + * @return end iterator of the table. + */ + int32_t db_idx64_end(uint64_t code, uint64_t scope, uint64_t table); + + /** + * Find the table row following the referenced table row in a secondary 64-bit integer index table. + * + * @ingroup database uint64_t-secondary-index + * @param iterator - the iterator to the referenced table row. + * @param[out] primary - pointer to a `uint64_t` variable which will have its value set to the primary key of the next table row. + * + * @return iterator to the table row following the referenced table row (or the end iterator of the table if the referenced table row is the last one in the table). + * @pre `iterator` points to an existing table row in the table. + * @post `*primary` will be replaced with the primary key of the table row following the referenced table row if it exists, otherwise `*primary` will be left untouched. + */ + int32_t db_idx64_next(int32_t iterator, legacy_ptr primary); + + /** + * Find the table row preceding the referenced table row in a secondary 64-bit integer index table. + * + * @ingroup database uint64_t-secondary-index + * @param iterator - the iterator to the referenced table row. + * @param[out] primary - pointer to a `uint64_t` variable which will have its value set to the primary key of the previous table row. + * + * @return iterator to the table row preceding the referenced table row assuming one exists (it will return -1 if the referenced table row is the first one in the table). + * @pre `iterator` points to an existing table row in the table or it is the end iterator of the table. + * @post `*primary` will be replaced with the primary key of the table row preceding the referenced table row if it exists, otherwise `*primary` will be left untouched. + */ + int32_t db_idx64_previous(int32_t iterator, legacy_ptr primary); + + /** + * Store an association of a 128-bit integer secondary key to a primary key in a secondary 128-bit integer index table. + * + * @ingroup database uint128_t-secondary-index + * @param scope - the scope where the table resides (implied to be within the code of the current receiver). + * @param table - the table name. + * @param payer - the account that is paying for this storage. + * @param id - the primary key to which to associate the secondary key. + * @param secondary - the pointer to the key of the secondary index to store. + * + * @return iterator to the newly created secondary index. + * @post new secondary key association between primary key `id` and secondary key `*secondary` is created in the secondary 128-bit integer index table. + */ + int32_t db_idx128_store(uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, legacy_ptr secondary); + + /** + * Update an association for a 128-bit integer secondary key to a primary key in a secondary 128-bit integer index table. + * + * @ingroup database uint128_t-secondary-index + * @param iterator - the iterator to the table row containing the secondary key association to update. + * @param payer - the account that pays for the storage costs. + * @param secondary - pointer to the **new** secondary key that will replace the existing one of the association. + * + * @pre `iterator` points to an existing table row in the table. + * @post the secondary key of the table row pointed to by `iterator` is replaced by `*secondary`. + */ + void db_idx128_update(int32_t iterator, uint64_t payer, legacy_ptr secondary); + + /** + * Remove a table row from a secondary 128-bit integer index table. + * + * @ingroup database uint128_t-secondary-index + * @param iterator - iterator to the table row to remove. + * + * @pre `iterator` points to an existing table row in the table. + * @post the table row pointed to by `iterator` is removed and the associated storage costs are refunded to the payer. + */ + void db_idx128_remove(int32_t iterator); + + /** + * Find a table row in a secondary 128-bit integer index table by secondary key. + * + * @ingroup database uint128_t-secondary-index + * + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * @param secondary - the pointer to the secondary index key. + * @param[out] primary - pointer to a 'uint64_t' variable which will have its value set to the primary key of the found table row. + * + * @return iterator to the first table row with a secondary key equal to `*secondary` or the end iterator of the table if the table row could not be found. + * @post If and only if the table row is found, `*primary` will be replaced with the primary key of the found table row. + */ + int32_t db_idx128_find_secondary(uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary); + + /** + * Find a table row in a secondary 128-bit integer index table by primary key. + * + * @ingroup database uint128_t-secondary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * @param[out] secondary - pointer to a 'uint128_t' variable which will have its value set to the secondary key of the found table row. + * @param primary - the primary key of the table row to look up. + * + * @return iterator to the table row with a primary key equal to `primary` or the end iterator of the table if the table row could not be found. + * @post If and only if the table row is found, `*secondary` will be replaced with the secondary key of the found table row. + */ + int32_t db_idx128_find_primary(uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, uint64_t primary); + + /** + * Find the table row in a secondary 128-bit integer index table that matches the lowerbound condition for a given secondary key. + * Lowerbound secondary index is the first secondary index which key is <= the given secondary index key. + * + * @ingroup database uint128_t-secondary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * @param[out] secondary - pointer to secondary key first used to determine the lowerbound and which is then replaced with the secondary key of the found table row. + * @param[out] primary - pointer to a `uint64_t` variable which will have its value set to the primary key of the found table row. + * + * @return iterator to the found table row or the end iterator of the table if the table row could not be found. + * + * @post If and only if the table row is found, `*secondary` will be replaced with the secondary key of the found table row. + * @post If and only if the table row is found, `*primary` will be replaced with the primary key of the found table row. + */ + int32_t db_idx128_lowerbound(uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary); + + /** + * Find the table row in a secondary 128-bit integer index table that matches the upperbound condition for a given secondary key. + * The table row that matches the upperbound condition is the first table row in the table with the lowest secondary key that is > the given key. + * + * @ingroup database uint128_t-secondary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * @param[out] secondary - pointer to secondary key first used to determine the upperbound and which is then replaced with the secondary key of the found table row. + * @param[out] primary - pointer to a `uint64_t` variable which will have its value set to the primary key of the found table row. + * + * @return iterator to the found table row or the end iterator of the table if the table row could not be found. + */ + int32_t db_idx128_upperbound(uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary); + + /** + * Get an end iterator representing just-past-the-end of the last table row of a secondary 128-bit integer index table. + * + * @ingroup database uint128_t-secondary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * + * @return end iterator of the table. + */ + int32_t db_idx128_end(uint64_t code, uint64_t scope, uint64_t table); + + /** + * Find the table row following the referenced table row in a secondary 128-bit integer index table. + * + * @ingroup database uint128_t-secondary-index + * @param iterator - the iterator to the referenced table row. + * @param[out] primary - pointer to a `uint64_t` variable which will have its value set to the primary key of the next table row. + * + * @return iterator to the table row following the referenced table row (or the end iterator of the table if the referenced table row is the last one in the table). + * @pre `iterator` points to an existing table row in the table. + * @post `*primary` will be replaced with the primary key of the table row following the referenced table row if it exists, otherwise `*primary` will be left untouched. + */ + int32_t db_idx128_next(int32_t iterator, legacy_ptr primary); + + /** + * Find the table row preceding the referenced table row in a secondary 128-bit integer index table. + * + * @ingroup database uint128_t-secondary-index + * @param iterator - the iterator to the referenced table row. + * @param[out] primary - pointer to a `uint64_t` variable which will have its value set to the primary key of the previous table row. + * + * @return iterator to the table row preceding the referenced table row assuming one exists (it will return -1 if the referenced table row is the first one in the table). + * @pre `iterator` points to an existing table row in the table or it is the end iterator of the table. + * @post `*primary` will be replaced with the primary key of the table row preceding the referenced table row if it exists, otherwise `*primary` will be left untouched. + */ + int32_t db_idx128_previous(int32_t iterator, legacy_ptr primary); + + /** + * Store an association of a 256-bit integer secondary key to a primary key in a secondary 256-bit integer index table. + * + * @ingroup database 256-bit-secondary-index + * @param scope - the scope where the table resides (implied to be within the code of the current receiver). + * @param table - the table name. + * @param payer - the account that is paying for this storage. + * @param id - the primary key to which to associate the secondary key. + * @param data - pointer to the secondary key data stored as an array of 2 `uint128_t` integers. + * + * @return iterator to the newly created secondary index. + * @post new secondary key association between primary key `id` and secondary key `*data` is created in the secondary 256-bit integer index table. + */ + int32_t db_idx256_store(uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, legacy_span data); + + /** + * Update an association for a 256-bit integer secondary key to a primary key in a secondary 256-bit integer index table. + * + * @ingroup database 256-bit-secondary-index + * @param iterator - the iterator to the table row containing the secondary key association to update. + * @param payer - the account that pays for the storage costs. + * @param data - pointer to the **new** secondary key data (which is stored as an array of 2 `uint128_t` integers) that will replace the existing one of the association. + * + * @pre `iterator` points to an existing table row in the table. + * @post the secondary key of the table row pointed to by `iterator` is replaced by the specified secondary key. + */ + void db_idx256_update(int32_t iterator, uint64_t payer, legacy_span data); + + /** + * Remove a table row from a secondary 256-bit integer index table. + * + * @ingroup database 256-bit-secondary-index + * @param iterator - iterator to the table row to remove. + * + * @pre `iterator` points to an existing table row in the table. + * @post the table row pointed to by `iterator` is removed and the associated storage costs are refunded to the payer. + */ + void db_idx256_remove(int32_t iterator); + + /** + * Find a table row in a secondary 256-bit integer index table by secondary key. + * + * @ingroup database 256-bit-secondary-index + * + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * @param data - pointer to the secondary key data (which is stored as an array of 2 `uint128_t` integers) used to lookup the table row. + * @param[out] primary - pointer to a 'uint64_t' variable which will have its value set to the primary key of the found table row. + * + * @return iterator to the first table row with a secondary key equal to the specified secondary key or the end iterator of the table if the table row could not be found. + * @post If and only if the table row is found, `*primary` will be replaced with the primary key of the found table row. + */ + int32_t db_idx256_find_secondary(uint64_t code, uint64_t scope, uint64_t table, legacy_span data, legacy_ptr primary); + + /** + * Find a table row in a secondary 256-bit integer index table by primary key. + * + * @ingroup database 256-bit-secondary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * @param[out] data - pointer to the array of 2 `uint128_t` integers which will act as the buffer to hold the retrieved secondary key of the found table row. + * @param primary - the primary key of the table row to look up. + * + * @return iterator to the table row with a primary key equal to `data` or the end iterator of the table if the table row could not be found. + * @post If and only if the table row is found, `data` will be replaced with the secondary key of the found table row. + */ + int32_t db_idx256_find_primary(uint64_t code, uint64_t scope, uint64_t table, legacy_span data, uint64_t primary); + + /** + * Find the table row in a secondary 256-bit integer index table that matches the lowerbound condition for a given secondary key. + * Lowerbound secondary index is the first secondary index which key is <= the given secondary index key. + * + * @ingroup database 256-bit-secondary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * @param[out] data - pointer to the secondary key data (which is stored as an array of 2 `uint128_t` integers) first used to determine the lowerbound and which is then replaced with the secondary key of the found table row. + * @param[out] primary - pointer to a `uint64_t` variable which will have its value set to the primary key of the found table row. + * + * @return iterator to the found table row or the end iterator of the table if the table row could not be found. + * + * @post If and only if the table row is found, `data` will be replaced with the secondary key of the found table row. + * @post If and only if the table row is found, `*primary` will be replaced with the primary key of the found table row. + */ + int32_t db_idx256_lowerbound(uint64_t code, uint64_t scope, uint64_t table, legacy_span data, legacy_ptr primary); + + /** + * Find the table row in a secondary 256-bit integer index table that matches the upperbound condition for a given secondary key. + * The table row that matches the upperbound condition is the first table row in the table with the lowest secondary key that is > the given key. + * + * @ingroup database 256-bit-secondary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * @param[out] data - pointer to the secondary key data (which is stored as an array of 2 `uint128_t` integers) first used to determine the upperbound and which is then replaced with the secondary key of the found table row. + * @param[out] primary - pointer to a `uint64_t` variable which will have its value set to the primary key of the found table row. + * + * @return iterator to the found table row or the end iterator of the table if the table row could not be found. + * + * @post If and only if the table row is found, the buffer pointed to by `data` will be filled with the secondary key of the found table row. + * @post If and only if the table row is found, `*primary` will be replaced with the primary key of the found table row. + */ + int32_t db_idx256_upperbound(uint64_t code, uint64_t scope, uint64_t table, legacy_span data, legacy_ptr primary); + + /** + * Get an end iterator representing just-past-the-end of the last table row of a secondary 256-bit integer index table. + * + * @ingroup database 256-bit-secondary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * + * @return end iterator of the table. + */ + int32_t db_idx256_end(uint64_t code, uint64_t scope, uint64_t table); + + /** + * Find the table row following the referenced table row in a secondary 256-bit integer index table. + * + * @ingroup database 256-bit-secondary-index + * @param iterator - the iterator to the referenced table row. + * @param[out] primary - pointer to a `uint64_t` variable which will have its value set to the primary key of the next table row. + * + * @return iterator to the table row following the referenced table row (or the end iterator of the table if the referenced table row is the last one in the table). + * @pre `iterator` points to an existing table row in the table. + * @post `*primary` will be replaced with the primary key of the table row following the referenced table row if it exists, otherwise `*primary` will be left untouched. + */ + int32_t db_idx256_next(int32_t iterator, legacy_ptr primary); + + /** + * Find the table row preceding the referenced table row in a secondary 256-bit integer index table. + * + * @ingroup database 256-bit-secondary-index + * @param iterator - the iterator to the referenced table row. + * @param[out] primary - pointer to a `uint64_t` variable which will have its value set to the primary key of the previous table row. + * + * @return iterator to the table row preceding the referenced table row assuming one exists (it will return -1 if the referenced table row is the first one in the table). + * @pre `iterator` points to an existing table row in the table or it is the end iterator of the table. + * @post `*primary` will be replaced with the primary key of the table row preceding the referenced table row if it exists, otherwise `*primary` will be left untouched. + */ + int32_t db_idx256_previous(int32_t iterator, legacy_ptr primary); + + /** + * Store an association of a double-precision floating-point secondary key to a primary key in a secondary double-precision floating-point index table. + * + * @ingroup database double-secondary-index + * @param scope - the scope where the table resides (implied to be within the code of the current receiver). + * @param table - the table name. + * @param payer - the account that is paying for this storage. + * @param id - the primary key to which to associate the secondary key. + * @param secondary - pointer to the secondary key. + * + * @return iterator to the newly created secondary index. + * @post new secondary key association between primary key `id` and secondary key `*secondary` is created in the secondary double-precision floating-point index table. + */ + int32_t db_idx_double_store(uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, legacy_ptr secondary); + + /** + * Update an association for a double-precision floating-point secondary key to a primary key in a secondary double-precision floating-point index table. + * + * @ingroup database double-secondary-index + * @param iterator - the iterator to the table row containing the secondary key association to update. + * @param payer - the account that pays for the storage costs. + * @param secondary - pointer to the **new** secondary key that will replace the existing one of the association. + * + * @pre `iterator` points to an existing table row in the table. + * @post the secondary key of the table row pointed to by `iterator` is replaced by the specified secondary key. + */ + void db_idx_double_update(int32_t iterator, uint64_t payer, legacy_ptr secondary); + + /** + * Remove a table row from a secondary double-precision floating-point index table. + * + * @ingroup database double-secondary-index + * @param iterator - iterator to the table row to remove. + * + * @pre `iterator` points to an existing table row in the table. + * @post the table row pointed to by `iterator` is removed and the associated storage costs are refunded to the payer. + */ + void db_idx_double_remove(int32_t iterator); + + /** + * Find a table row in a secondary double-precision floating-point index table by secondary key. + * + * @ingroup database double-secondary-index + * + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * @param secondary - Pointer to secondary key used to lookup the table row. + * @param[out] primary - pointer to a 'uint64_t' variable which will have its value set to the primary key of the found table row. + * + * @return iterator to the first table row with a secondary key equal to the specified secondary key or the end iterator of the table if the table row could not be found. + * @post If and only if the table row is found, `*primary` will be replaced with the primary key of the found table row. + */ + int32_t db_idx_double_find_secondary(uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary); + + /** + * Find a table row in a secondary double-precision floating-point index table by primary key. + * + * @ingroup database double-secondary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * @param[out] secondary - pointer to a `double` variable which will have its value set to the secondary key of the found table row. + * @param primary - the primary key of the table row to look up. + * + * @return iterator to the table row with a primary key equal to `secondary` or the end iterator of the table if the table row could not be found. + * @post If and only if the table row is found, `secondary` will be replaced with the secondary key of the found table row. + */ + int32_t db_idx_double_find_primary(uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, uint64_t primary); + + /** + * Find the table row in a secondary double-precision floating-point index table that matches the lowerbound condition for a given secondary key. + * Lowerbound secondary index is the first secondary index which key is <= the given secondary index key. + * + * @ingroup database double-secondary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * @param[out] secondary - Pointer to secondary key first used to determine the lowerbound and which is then replaced with the secondary key of the found table row. + * @param[out] primary - pointer to a `uint64_t` variable which will have its value set to the primary key of the found table row. + * + * @return iterator to the found table row or the end iterator of the table if the table row could not be found. + * + * @post If and only if the table row is found, `*secondary` will be replaced with the secondary key of the found table row. + * @post If and only if the table row is found, `*primary` will be replaced with the primary key of the found table row. + */ + int32_t db_idx_double_lowerbound(uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary); + + /** + * Find the table row in a secondary double-precision floating-point index table that matches the upperbound condition for a given secondary key. + * The table row that matches the upperbound condition is the first table row in the table with the lowest secondary key that is > the given key. + * + * @ingroup database double-secondary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * @param[out] secondary - pointer to secondary key first used to determine the upperbound and which is then replaced with the secondary key of the found table row. + * @param[out] primary - pointer to a `uint64_t` variable which will have its value set to the primary key of the found table row. + * + * @return iterator to the found table row or the end iterator of the table if the table row could not be found. + * + * @post If and only if the table row is found, the buffer pointed to by `*secondary` will be filled with the secondary key of the found table row. + * @post If and only if the table row is found, `*primary` will be replaced with the primary key of the found table row. + */ + int32_t db_idx_double_upperbound(uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary); + + /** + * Get an end iterator representing just-past-the-end of the last table row of a secondary double-precision floating-point index table. + * + * @ingroup database double-secondary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * + * @return end iterator of the table. + */ + int32_t db_idx_double_end(uint64_t code, uint64_t scope, uint64_t table); + + /** + * Find the table row following the referenced table row in a secondary double-precision floating-point index table. + * + * @ingroup database double-secondary-index + * @param iterator - the iterator to the referenced table row. + * @param[out] primary - pointer to a `uint64_t` variable which will have its value set to the primary key of the next table row. + * + * @return iterator to the table row following the referenced table row (or the end iterator of the table if the referenced table row is the last one in the table). + * @pre `iterator` points to an existing table row in the table. + * @post `*primary` will be replaced with the primary key of the table row following the referenced table row if it exists, otherwise `*primary` will be left untouched. + */ + int32_t db_idx_double_next(int32_t iterator, legacy_ptr primary); + + /** + * Find the table row preceding the referenced table row in a secondary double-precision floating-point index table. + * + * @ingroup database double-secondary-index + * @param iterator - the iterator to the referenced table row. + * @param[out] primary - pointer to a `uint64_t` variable which will have its value set to the primary key of the previous table row. + * + * @return iterator to the table row preceding the referenced table row assuming one exists (it will return -1 if the referenced table row is the first one in the table). + * @pre `iterator` points to an existing table row in the table or it is the end iterator of the table. + * @post `*primary` will be replaced with the primary key of the table row preceding the referenced table row if it exists, otherwise `*primary` will be left untouched. + */ + int32_t db_idx_double_previous(int32_t iterator, legacy_ptr primary); + + /** + * Store an association of a quadruple-precision floating-point secondary key to a primary key in a secondary quadruple-precision floating-point index table. + * + * @ingroup database long-double-secondary-index + * @param scope - the scope where the table resides (implied to be within the code of the current receiver). + * @param table - the table name. + * @param payer - the account that is paying for this storage. + * @param id - the primary key to which to associate the secondary key. + * @param secondary - pointer to the secondary key. + * + * @return iterator to the newly created secondary index. + * @post new secondary key association between primary key `id` and secondary key `*secondary` is created in the quadruple-precision floating-point index table. + */ + int32_t db_idx_long_double_store(uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, legacy_ptr secondary); + + /** + * Update an association for a quadruple-precision floating-point secondary key to a primary key in a secondary quadruple-precision floating-point index table. + * + * @ingroup database long-double-secondary-index + * @param iterator - the iterator to the table row containing the secondary key association to update. + * @param payer - the account that pays for the storage costs. + * @param secondary - pointer to the **new** secondary key that will replace the existing one of the association. + * + * @pre `iterator` points to an existing table row in the table. + * @post the secondary key of the table row pointed to by `iterator` is replaced by the specified secondary key. + */ + void db_idx_long_double_update(int32_t iterator, uint64_t payer, legacy_ptr secondary); + + /** + * Remove a table row from a secondary quadruple-precision floating-point index table. + * + * @ingroup database long-double-secondary-index + * @param iterator - iterator to the table row to remove. + * + * @pre `iterator` points to an existing table row in the table. + * @post the table row pointed to by `iterator` is removed and the associated storage costs are refunded to the payer. + */ + void db_idx_long_double_remove(int32_t iterator); + + /** + * Find a table row in a secondary quadruple-precision floating-point index table by secondary key. + * + * @ingroup database long-double-secondary-index + * + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * @param secondary - Pointer to secondary key used to lookup the table row. + * @param[out] primary - pointer to a 'uint64_t' variable which will have its value set to the primary key of the found table row. + * + * @return iterator to the first table row with a secondary key equal to the specified secondary key or the end iterator of the table if the table row could not be found. + * @post If and only if the table row is found, `*primary` will be replaced with the primary key of the found table row. + */ + int32_t db_idx_long_double_find_secondary(uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary); + + /** + * Find a table row in a secondary double-precision floating-point index table by primary key. + * + * @ingroup database long-double-secondary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * @param[out] secondary - pointer to a `long double` variable which will have its value set to the secondary key of the found table row. + * @param primary - the primary key of the table row to look up. + * + * @return iterator to the table row with a primary key equal to `secondary` or the end iterator of the table if the table row could not be found. + * @post If and only if the table row is found, `secondary` will be replaced with the secondary key of the found table row. + */ + int32_t db_idx_long_double_find_primary(uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, uint64_t primary); + + /** + * Find the table row in a secondary quadruple-precision floating-point index table that matches the lowerbound condition for a given secondary key. + * Lowerbound secondary index is the first secondary index which key is <= the given secondary index key. + * + * @ingroup database long-double-secondary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * @param[out] secondary - Pointer to secondary key first used to determine the lowerbound and which is then replaced with the secondary key of the found table row. + * @param[out] primary - pointer to a `uint64_t` variable which will have its value set to the primary key of the found table row. + * + * @return iterator to the found table row or the end iterator of the table if the table row could not be found. + * + * @post If and only if the table row is found, `*secondary` will be replaced with the secondary key of the found table row. + * @post If and only if the table row is found, `*primary` will be replaced with the primary key of the found table row. + */ + int32_t db_idx_long_double_lowerbound(uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary); + + /** + * Find the table row in a secondary quadruple-precision floating-point index table that matches the upperbound condition for a given secondary key. + * The table row that matches the upperbound condition is the first table row in the table with the lowest secondary key that is > the given key. + * + * @ingroup database long-double-secondary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * @param[out] secondary - pointer to secondary key first used to determine the upperbound and which is then replaced with the secondary key of the found table row. + * @param[out] primary - pointer to a `uint64_t` variable which will have its value set to the primary key of the found table row. + * + * @return iterator to the found table row or the end iterator of the table if the table row could not be found. + * + * @post If and only if the table row is found, the buffer pointed to by `*secondary` will be filled with the secondary key of the found table row. + * @post If and only if the table row is found, `*primary` will be replaced with the primary key of the found table row. + */ + int32_t db_idx_long_double_upperbound(uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary); + + /** + * Get an end iterator representing just-past-the-end of the last table row of a secondary quadruple-precision floating-point index table. + * + * @ingroup database long-double-secondary-index + * @param code - the name of the owner of the table. + * @param scope - the scope where the table resides. + * @param table - the table name. + * + * @return end iterator of the table. + */ + int32_t db_idx_long_double_end(uint64_t code, uint64_t scope, uint64_t table); + + /** + * Find the table row following the referenced table row in a secondary quadruple-precision floating-point index table. + * + * @ingroup database long-double-secondary-index + * @param iterator - the iterator to the referenced table row. + * @param[out] primary - pointer to a `uint64_t` variable which will have its value set to the primary key of the next table row. + * + * @return iterator to the table row following the referenced table row (or the end iterator of the table if the referenced table row is the last one in the table). + * @pre `iterator` points to an existing table row in the table. + * @post `*primary` will be replaced with the primary key of the table row following the referenced table row if it exists, otherwise `*primary` will be left untouched. + */ + int32_t db_idx_long_double_next(int32_t iterator, legacy_ptr primary); + + /** + * Find the table row preceding the referenced table row in a secondary quadruple-precision floating-point index table. + * + * @ingroup database long-double-secondary-index + * @param iterator - the iterator to the referenced table row. + * @param[out] primary - pointer to a `uint64_t` variable which will have its value set to the primary key of the previous table row. + * + * @return iterator to the table row preceding the referenced table row assuming one exists (it will return -1 if the referenced table row is the first one in the table). + * @pre `iterator` points to an existing table row in the table or it is the end iterator of the table. + * @post `*primary` will be replaced with the primary key of the table row preceding the referenced table row if it exists, otherwise `*primary` will be left untouched. + */ + int32_t db_idx_long_double_previous(int32_t iterator, legacy_ptr primary); + + /** + * Erase a key-value pair. + * + * @ingroup kv-database + * @param contract - name of the contract associated with the kv pair. + * @param key - the key associated with the kv pair to be erased. + * + * @return change in resource usage. + */ + int64_t kv_erase(uint64_t contract, span key); + + /** + * Set a key-value pair. + * If the key doesn't exist, then a new kv pair will be created. + * + * @ingroup kv-database + * @param contract - name of the contract associated with the kv pair. + * @param key - the key in the kv pair. + * @param value - the value in the kv pair. + * @param payer - name of the account paying for the resource. + * + * @return change in resource usage. + */ + int64_t kv_set(uint64_t contract, span key, span value, account_name payer); + + /** + * Check the existence of a key. + * If the key doesn't exist, it returns false, clears the temporary data buffer and sets *value_size to 0. + * If the key does exist, it returns true, stores the value into the temporary data buffer + * and sets *value_size to the value size. + * Use kv_get_data to retrieve the value. + * + * @ingroup kv-database + * @param contract - name of the contract associated with the kv pair. + * @param key - the key to query. + * @param[out] value_size - resulting value size (0 if the key doesn't exist). + * + * @return false if the provided key doesn't exist, true otherwise. + */ + bool kv_get(uint64_t contract, span key, uint32_t* value_size); + + /** + * Fetches data from temporary buffer starting at offset. + * Copies up to the data's size span passed as parameter. + * + * @ingroup kv-database + * @param offset - position from where to start reading the value from the temporary buffer. + * @param[out] data - span where the result value will be stored. + * + * @return number of bytes written in data. + */ + uint32_t kv_get_data(uint32_t offset, span data); + + /** + * Create a kv iterator. + * The returned handle: + * - Starts at 1. + * - Counts up by 1 at each call, assuming no destroyed iterators are available. + * - If destroyed iterators are available, the most-recently-destroyed one is reinitialized and returned. + * The prefix limits the range of keys that the iterator covers. If the prefix is empty, + * the iterator covers the entire range of keys belonging to a contract within the database ID. + * + * @ingroup kv-database + * @param contract - the contract associated with the kv iterator. + * @param prefix - prefix to build the iterator. + * + * @return handle of the created iterator + */ + uint32_t kv_it_create(uint64_t contract, span prefix); + + /** + * Destroy a kv iterator. + * + * @ingroup kv-database + * @param itr - the kv iterator to destroy. + */ + void kv_it_destroy(uint32_t itr); + + /** + * Get the status of a kv iterator. + * + * @ingroup kv-database + * @param itr - the kv iterator we want to know the status. + * + * @retval iterator_ok iterator is positioned at a kv pair. + * @retval iterator_erased the kv pair that the iterator used to be positioned at, was erased. + * @retval iterator_end iterator is out-of-bounds. + */ + int32_t kv_it_status(uint32_t itr); + + /** + * Compare the key of two kv iterators. + * + * @ingroup kv-database + * @param itr_a - first iterator. + * @param itr_b - second iterator. + * + * @retval -1 if itr_a's key is less than itr_b's key + * @retval 0 if itr_a's key is the same as itr_b's key + * @retval 1 if itr_a's key is greater than itr_b's key + */ + int32_t kv_it_compare(uint32_t itr_a, uint32_t itr_b); + + /** + * Compare the key of an iterator to a provided key. + * + * @ingroup kv-database + * @param itr - the iterator to compare. + * @param key - the key to compare. + * + * @retval -1 if itr's key is less than key. + * @retval 0 if itr's key is the same as key. + * @retval 1 itr's key is greater than key. + */ + int32_t kv_it_key_compare(uint32_t itr, span key); + + /** + * Move a kv iterator to an out-of-bounds position. + * + * @ingroup kv-database + * @param itr - the kv iterator we want to move. + * + * @return the status of the iterator (iterator_end). + */ + int32_t kv_it_move_to_end(uint32_t itr); + + /** + * Move a kv iterator to the next position. + * + * @ingroup kv-database + * @param itr - the iterator we want to move. + * @param[out] found_key_size - size of the result key in the new position. + * @param[out] found_value_size - size of the result value in the new position. + * + * @return the status of the iterator. + */ + int32_t kv_it_next(uint32_t itr, uint32_t* found_key_size, uint32_t* found_value_size); + + /** + * Move a kv iterator to the previous position. + * + * @ingroup kv-database + * @param itr - the iterator we want to move + * @param[out] found_key_size - size of the result key in the new position. + * @param[out] found_value_size - size of the result value in the new position. + * + * @return the status of the iterator. + */ + int32_t kv_it_prev(uint32_t itr, uint32_t* found_key_size, uint32_t* found_value_size); + + /** + * Find the least non-deleted key which is >= the provided key. + * + * @ingroup kv-database + * @param itr - the kv iterator. + * @param key - the key we want to query + * @param[out] found_key_size - size of the result found key + * @param[out] found_value_size - size of the result found value + * + * @return the status of the iterator. + * @post if a key is found, the new status is iterator_ok. If not found, the new status is iterator_end. kv_it_lower_bound never returns iterator_erased. + * @post the size of the key and the size of the value at the iterator's new position, or zero if the iterator is at end, will be written to found_key_size and found_value_size. + */ + int32_t kv_it_lower_bound(uint32_t itr, span key, uint32_t* found_key_size, uint32_t* found_value_size); + + /** + * Fetch the key from a kv iterator. + * + * @ingroup kv-database + * @param itr - the kv iterator. + * @param offset - position from where to start reading. + * @param[out] dest - where the data will reside. + * @param[out] actual_size - size of the key + * + * @return the status of the iterator passed as a parameter. + */ + int32_t kv_it_key(uint32_t itr, uint32_t offset, span dest, uint32_t* actual_size); + + /** + * Fetch the value from a kv iterator. + * + * @ingroup kv-database + * @param itr - the iterator of the object we want to get the value. + * @param offset - position from where to start reading. + * @param[out] dest - where the value will reside. + * @param[out] actual_size - size of the value. + * + * @return the status of the iterator passed as a parameter. + */ + int32_t kv_it_value(uint32_t itr, uint32_t offset, span dest, uint32_t* actual_size); + + // memory api + void* memcpy(memcpy_params) const; + void* memmove(memcpy_params) const; + int32_t memcmp(memcmp_params) const; + void* memset(memset_params) const; + + /** + * Send an inline action in the context of the parent transaction of this operation. + * + * @ingroup transaction + * @param data - the inline action to be sent. + */ + void send_inline(legacy_span data); + + /** + * Send a context free inline action in the context of the parent transaction of this operation. + * + * @ingroup transaction + * @param data - the packed free inline action to be sent. + */ + void send_context_free_inline(legacy_span data); + + /** + * Send a deferred transaction. + * + * @ingroup transaction + * @param sender_id - account name of the sender of this deferred transaction. + * @param payer - account name responsible for paying the RAM for this deferred transaction. + * @param data - the packed transaction to be deferred. + * @param replace_existing - if true, it will replace an existing transaction. + */ + void send_deferred(legacy_ptr sender_id, account_name payer, legacy_span data, uint32_t replace_existing); + + /** + * Cancels a deferred transaction. + * + * @ingroup transaction + * @param val - The id of the sender. + * + * @retval false if transaction was not found. + * @retval true if transaction was canceled. + */ + bool cancel_deferred(legacy_ptr val); + + /** + * Access a copy of the currently executing transaction. + * + * @ingroup context-free-transaction + * @param[out] data - the currently executing transaction (packed). + * + * @retval false if transaction was not found. + * @retval true if transaction was canceled. + */ + int32_t read_transaction(legacy_span data) const; + + /** + * Gets the size of the currently executing transaction. + * + * @ingroup context-free-transaction + * + * @return size of the currently executing transaction. + */ + int32_t transaction_size() const; + + /** + * Gets the expiration of the currently executing transaction. + * + * @ingroup context-free-transaction + * + * @return expiration of the currently executing transaction in seconds since Unix epoch. + */ + int32_t expiration() const; + + /** + * Gets the block number used for TAPOS on the currently executing transaction. + * + * @ingroup context-free-transaction + * + * @return block number used for TAPOS on the currently executing transaction. + */ + int32_t tapos_block_num() const; + + /** + * Gets the block prefix used for TAPOS on the currently executing transaction. + * + * @ingroup context-free-transaction + * + * @return block prefix used for TAPOS on the currently executing transaction. + */ + int32_t tapos_block_prefix() const; + + /** + * Retrieve the indicated action from the active transaction. + * + * @ingroup context-free-transaction + * + * @param type - 0 for context free action, 1 for action. + * @param index - the index of the requested action. + * @param[out] buffer - the action we want (packed). + * + * @return the number of bytes written on the buffer or -1 if there was an error. + */ + int32_t get_action(uint32_t type, uint32_t index, legacy_span buffer) const; + + // compiler builtins api + void __ashlti3(legacy_ptr, uint64_t, uint64_t, uint32_t) const; + void __ashrti3(legacy_ptr, uint64_t, uint64_t, uint32_t) const; + void __lshlti3(legacy_ptr, uint64_t, uint64_t, uint32_t) const; + void __lshrti3(legacy_ptr, uint64_t, uint64_t, uint32_t) const; + void __divti3(legacy_ptr, uint64_t, uint64_t, uint64_t, uint64_t) const; + void __udivti3(legacy_ptr, uint64_t, uint64_t, uint64_t, uint64_t) const; + void __multi3(legacy_ptr, uint64_t, uint64_t, uint64_t, uint64_t) const; + void __modti3(legacy_ptr, uint64_t, uint64_t, uint64_t, uint64_t) const; + void __umodti3(legacy_ptr, uint64_t, uint64_t, uint64_t, uint64_t) const; + void __addtf3(legacy_ptr, uint64_t, uint64_t, uint64_t, uint64_t) const; + void __subtf3(legacy_ptr, uint64_t, uint64_t, uint64_t, uint64_t) const; + void __multf3(legacy_ptr, uint64_t, uint64_t, uint64_t, uint64_t) const; + void __divtf3(legacy_ptr, uint64_t, uint64_t, uint64_t, uint64_t) const; + void __negtf2(legacy_ptr, uint64_t, uint64_t) const; + void __extendsftf2(legacy_ptr, float) const; + void __extenddftf2(legacy_ptr, double) const; + double __trunctfdf2(uint64_t, uint64_t) const; + float __trunctfsf2(uint64_t, uint64_t) const; + int32_t __fixtfsi(uint64_t, uint64_t) const; + int64_t __fixtfdi(uint64_t, uint64_t) const; + void __fixtfti(legacy_ptr, uint64_t, uint64_t) const; + uint32_t __fixunstfsi(uint64_t, uint64_t) const; + uint64_t __fixunstfdi(uint64_t, uint64_t) const; + void __fixunstfti(legacy_ptr, uint64_t, uint64_t) const; + void __fixsfti(legacy_ptr, float) const; + void __fixdfti(legacy_ptr, double) const; + void __fixunssfti(legacy_ptr, float) const; + void __fixunsdfti(legacy_ptr, double) const; + double __floatsidf(int32_t) const; + void __floatsitf(legacy_ptr, int32_t) const; + void __floatditf(legacy_ptr, uint64_t) const; + void __floatunsitf(legacy_ptr, uint32_t) const; + void __floatunditf(legacy_ptr, uint64_t) const; + double __floattidf(uint64_t, uint64_t) const; + double __floatuntidf(uint64_t, uint64_t) const; + int32_t __cmptf2(uint64_t, uint64_t, uint64_t, uint64_t) const; + int32_t __eqtf2(uint64_t, uint64_t, uint64_t, uint64_t) const; + int32_t __netf2(uint64_t, uint64_t, uint64_t, uint64_t) const; + int32_t __getf2(uint64_t, uint64_t, uint64_t, uint64_t) const; + int32_t __gttf2(uint64_t, uint64_t, uint64_t, uint64_t) const; + int32_t __letf2(uint64_t, uint64_t, uint64_t, uint64_t) const; + int32_t __lttf2(uint64_t, uint64_t, uint64_t, uint64_t) const; + int32_t __unordtf2(uint64_t, uint64_t, uint64_t, uint64_t) const; + + private: + apply_context& context; + }; + +}}} // ns eosio::chain::webassembly diff --git a/libraries/chain/include/eosio/chain/webassembly/preconditions.hpp b/libraries/chain/include/eosio/chain/webassembly/preconditions.hpp new file mode 100644 index 00000000000..3416557904b --- /dev/null +++ b/libraries/chain/include/eosio/chain/webassembly/preconditions.hpp @@ -0,0 +1,170 @@ +#pragma once + +#include +#include +#include + +#include +#include + +namespace eosio { namespace chain { namespace webassembly { + namespace detail { + template + constexpr std::integral_constant is_legacy_ptr(legacy_ptr); + template + constexpr std::false_type is_legacy_ptr(T); + template + constexpr std::integral_constant is_legacy_span(legacy_span); + template + constexpr std::false_type is_legacy_span(T); + + template + inline constexpr bool is_softfloat_type_v = + std::is_same_v || std::is_same_v || std::is_same_v; + + template + inline constexpr bool is_wasm_arithmetic_type_v = + is_softfloat_type_v || std::is_integral_v; + + template + struct is_whitelisted_legacy_type { + static constexpr bool value = std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v())), std::true_type> || + std::is_same_v())), std::true_type> || + std::is_same_v || + std::is_arithmetic_v; + }; + + template + struct is_whitelisted_type { + static constexpr bool value = is_wasm_arithmetic_type_v || + std::is_same_v; + }; + template + struct is_whitelisted_type> { + // Currently only a span of [const] char is allowed so there are no alignment concerns. + // If we wish to expand to general span in the future, changes are needed in EOS VM + // to check proper alignment of the void* within from_wasm before constructing the span. + static constexpr bool value = std::is_same_v, char>; + }; + template + struct is_whitelisted_type> { + static constexpr bool value = is_wasm_arithmetic_type_v>; + }; + } + + template + inline static constexpr bool is_whitelisted_type_v = detail::is_whitelisted_type::value; + + template + inline static constexpr bool is_whitelisted_legacy_type_v = detail::is_whitelisted_legacy_type::value; + + template + inline static constexpr bool are_whitelisted_legacy_types_v = (... && detail::is_whitelisted_legacy_type::value); + + template + inline static bool is_aliasing(const T& s1, const U& s2) { + std::uintptr_t a_begin = reinterpret_cast(s1.data()); + std::uintptr_t a_end = a_begin + s1.size_bytes(); + + std::uintptr_t b_begin = reinterpret_cast(s2.data()); + std::uintptr_t b_end = b_begin + s2.size_bytes(); + + // Aliasing iff the intersection of intervals [a_begin, a_end) and [b_begin, b_end) has non-zero size. + + if (a_begin > b_begin) { + std::swap(a_begin, b_begin); + std::swap(a_end, b_end); + } + + if (a_end <= b_begin) // intersection interval with non-zero size does not exist + return false; + + // Intersection interval is [b_begin, std::min(a_end, b_end)). + + if (std::min(a_end, b_end) == b_begin) // intersection interval has zero size + return false; + + return true; + } + + inline static bool is_nan( const float32_t f ) { + return f32_is_nan( f ); + } + inline static bool is_nan( const float64_t f ) { + return f64_is_nan( f ); + } + inline static bool is_nan( const float128_t& f ) { + return f128_is_nan( f ); + } + + EOS_VM_PRECONDITION(context_free_check, + EOS_VM_INVOKE_ONCE([&](auto&&...) { + EOS_ASSERT(ctx.get_host().get_context().is_context_free(), unaccessible_api, "this API may only be called from context_free apply"); + })); + + EOS_VM_PRECONDITION(context_aware_check, + EOS_VM_INVOKE_ONCE([&](auto&&...) { + EOS_ASSERT(!ctx.get_host().get_context().is_context_free(), unaccessible_api, "only context free api's can be used in this context"); + })); + + EOS_VM_PRECONDITION(privileged_check, + EOS_VM_INVOKE_ONCE([&](auto&&...) { + EOS_ASSERT(ctx.get_host().get_context().is_privileged(), unaccessible_api, + "${code} does not have permission to call this API", ("code", ctx.get_host().get_context().get_receiver())); + })); + + namespace detail { + template + vm::span to_span(const vm::argument_proxy& val) { + return {static_cast(val.get_original_pointer()), sizeof(T)}; + } + + template + vm::span to_span(const vm::span& val) { return val; } + } + + EOS_VM_PRECONDITION(core_precondition, + EOS_VM_INVOKE_ON_ALL(([&](auto&& arg, auto&&... rest) { + using namespace eosio::vm; + using arg_t = std::decay_t; + static_assert( is_whitelisted_type_v, "whitelisted type violation"); + if constexpr (is_span_type_v || vm::is_argument_proxy_type_v) { + eosio::vm::invoke_on([&arg](auto&& narg, auto&&... nrest) { + using nested_arg_t = std::decay_t; + if constexpr (eosio::vm::is_span_type_v || vm::is_argument_proxy_type_v) + EOS_ASSERT(!is_aliasing(detail::to_span(arg), detail::to_span(narg)), wasm_exception, "pointers not allowed to alias"); + }, rest...); + } + }))); + + template + inline constexpr bool should_check_nan_v = + std::is_same_v || std::is_same_v || std::is_same_v; + + template + struct remove_argument_proxy { + using type = T; + }; + template + struct remove_argument_proxy> { + using type = T; + }; + + EOS_VM_PRECONDITION(is_nan_check, + EOS_VM_INVOKE_ON_ALL([&](auto&& arg, auto&&... rest) { + if constexpr (should_check_nan_v>::type>>) { + EOS_ASSERT(!webassembly::is_nan(*arg), transaction_exception, "NaN is not an allowed value for a secondary key"); + } + })); + + EOS_VM_PRECONDITION(legacy_static_check_wl_args, + EOS_VM_INVOKE_ONCE([&](auto&&... args) { + static_assert( are_whitelisted_legacy_types_v...>, "legacy whitelisted type violation"); + })); + +}}} // ns eosio::chain::webassembly diff --git a/libraries/chain/include/eosio/chain/webassembly/wabt.hpp b/libraries/chain/include/eosio/chain/webassembly/wabt.hpp deleted file mode 100644 index ac0339fdfc2..00000000000 --- a/libraries/chain/include/eosio/chain/webassembly/wabt.hpp +++ /dev/null @@ -1,747 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -//wabt includes -#include -#include -#include - -namespace eosio { namespace chain { namespace webassembly { namespace wabt_runtime { - -using namespace fc; -using namespace wabt; -using namespace wabt::interp; -using namespace eosio::chain::webassembly::common; - -struct wabt_apply_instance_vars { - Memory* memory; - apply_context& ctx; - - char* get_validated_pointer(uint32_t offset, uint32_t size) { - EOS_ASSERT(memory, wasm_execution_error, "access violation"); - EOS_ASSERT(offset + size <= memory->data.size() && offset + size >= offset, wasm_execution_error, "access violation"); - return memory->data.data() + offset; - } -}; - -struct intrinsic_registrator { - using intrinsic_fn = TypedValue(*)(wabt_apply_instance_vars&, const TypedValues&); - - struct intrinsic_func_info { - FuncSignature sig; - intrinsic_fn func; - }; - - static auto& get_map(){ - static map> _map; - return _map; - }; - - intrinsic_registrator(const char* mod, const char* name, const FuncSignature& sig, intrinsic_fn fn) { - get_map()[string(mod)][string(name)] = intrinsic_func_info{sig, fn}; - } -}; - -class wabt_runtime : public eosio::chain::wasm_runtime_interface { - public: - wabt_runtime(); - bool inject_module(IR::Module&) override; - std::unique_ptr instantiate_module(const char* code_bytes, size_t code_size, std::vector initial_memory, - const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override; - - void immediately_exit_currently_running_module() override; - - private: - wabt::ReadBinaryOptions read_binary_options; //note default ctor will look at each option in feature.def and default to DISABLED for the feature -}; - -/** - * class to represent an in-wasm-memory array - * it is a hint to the transcriber that the next parameter will - * be a size (data bytes length) and that the pair are validated together - * This triggers the template specialization of intrinsic_invoker_impl - * @tparam T - */ -template -inline array_ptr array_ptr_impl (wabt_apply_instance_vars& vars, uint32_t ptr, uint32_t length) -{ - EOS_ASSERT( length < INT_MAX/(uint32_t)sizeof(T), binaryen_exception, "length will overflow" ); - return array_ptr((T*)(vars.get_validated_pointer(ptr, length * (uint32_t)sizeof(T)))); -} - -/** - * class to represent an in-wasm-memory char array that must be null terminated - */ -inline null_terminated_ptr null_terminated_ptr_impl(wabt_apply_instance_vars& vars, uint32_t ptr) -{ - char *value = vars.get_validated_pointer(ptr, 1); - const char* p = value; - const char* const top_of_memory = vars.memory->data.data() + vars.memory->data.size(); - while(p < top_of_memory) - if(*p++ == '\0') - return null_terminated_ptr(value); - - FC_THROW_EXCEPTION(wasm_execution_error, "unterminated string"); -} - - -template -struct is_reference_from_value { - static constexpr bool value = false; -}; - -template<> -struct is_reference_from_value { - static constexpr bool value = true; -}; - -template<> -struct is_reference_from_value { - static constexpr bool value = true; -}; - -template -constexpr bool is_reference_from_value_v = is_reference_from_value::value; - -template -T convert_literal_to_native(const TypedValue& v); - -template<> -inline double convert_literal_to_native(const TypedValue& v) { - return v.get_f64(); -} - -template<> -inline float convert_literal_to_native(const TypedValue& v) { - return v.get_f32(); -} - -template<> -inline int64_t convert_literal_to_native(const TypedValue& v) { - return v.get_i64(); -} - -template<> -inline uint64_t convert_literal_to_native(const TypedValue& v) { - return v.get_i64(); -} - -template<> -inline int32_t convert_literal_to_native(const TypedValue& v) { - return v.get_i32(); -} - -template<> -inline uint32_t convert_literal_to_native(const TypedValue& v) { - return v.get_i32(); -} - -template<> -inline bool convert_literal_to_native(const TypedValue& v) { - return v.get_i32(); -} - -template<> -inline name convert_literal_to_native(const TypedValue& v) { - int64_t val = v.get_i64(); - return name(val); -} - -inline auto convert_native_to_literal(const wabt_apply_instance_vars&, const uint32_t &val) { - TypedValue tv(Type::I32); - tv.set_i32(val); - return tv; -} - -inline auto convert_native_to_literal(const wabt_apply_instance_vars&, const int32_t &val) { - TypedValue tv(Type::I32); - tv.set_i32(val); - return tv; -} - -inline auto convert_native_to_literal(const wabt_apply_instance_vars&, const uint64_t &val) { - TypedValue tv(Type::I64); - tv.set_i64(val); - return tv; -} - -inline auto convert_native_to_literal(const wabt_apply_instance_vars&, const int64_t &val) { - TypedValue tv(Type::I64); - tv.set_i64(val); - return tv; -} - -inline auto convert_native_to_literal(const wabt_apply_instance_vars&, const float &val) { - TypedValue tv(Type::F32); - tv.set_f32(val); - return tv; -} - -inline auto convert_native_to_literal(const wabt_apply_instance_vars&, const double &val) { - TypedValue tv(Type::F64); - tv.set_f64(val); - return tv; -} - -inline auto convert_native_to_literal(const wabt_apply_instance_vars&, const name &val) { - TypedValue tv(Type::I64); - tv.set_i64(val.to_uint64_t()); - return tv; -} - -inline auto convert_native_to_literal(const wabt_apply_instance_vars& vars, char* ptr) { - const char* base = vars.memory->data.data(); - const char* top_of_memory = base + vars.memory->data.size(); - EOS_ASSERT(ptr >= base && ptr < top_of_memory, wasm_execution_error, "returning pointer not in linear memory"); - Value v; - v.i32 = (int)(ptr - base); - return TypedValue(Type::I32, v); -} - -struct void_type { -}; - -template -struct wabt_to_value_type; - -template<> -struct wabt_to_value_type { - static constexpr auto value = Type::F32; -}; - -template<> -struct wabt_to_value_type { - static constexpr auto value = Type::F64; -}; -template<> -struct wabt_to_value_type { - static constexpr auto value = Type::I32; -}; -template<> -struct wabt_to_value_type { - static constexpr auto value = Type::I64; -}; - -template -constexpr auto wabt_to_value_type_v = wabt_to_value_type::value; - -template -struct wabt_to_rvalue_type; -template<> -struct wabt_to_rvalue_type { - static constexpr auto value = Type::F32; -}; -template<> -struct wabt_to_rvalue_type { - static constexpr auto value = Type::F64; -}; -template<> -struct wabt_to_rvalue_type { - static constexpr auto value = Type::I32; -}; -template<> -struct wabt_to_rvalue_type { - static constexpr auto value = Type::I64; -}; -template<> -struct wabt_to_rvalue_type { - static constexpr auto value = Type::I64; -}; -template<> -struct wabt_to_rvalue_type { - static constexpr auto value = Type::I64; -}; - -template<> -struct wabt_to_rvalue_type { - static constexpr auto value = Type::I32; -}; - -template -constexpr auto wabt_to_rvalue_type_v = wabt_to_rvalue_type::value; - -template -struct wabt_function_type_provider; - -template -struct wabt_function_type_provider { - static FuncSignature type() { - return FuncSignature({wabt_to_value_type_v ...}, {wabt_to_rvalue_type_v}); - } -}; -template -struct wabt_function_type_provider { - static FuncSignature type() { - return FuncSignature({wabt_to_value_type_v ...}, {}); - } -}; - -/** - * Forward declaration of the invoker type which transcribes arguments to/from a native method - * and injects the appropriate checks - * - * @tparam Ret - the return type of the native function - * @tparam NativeParameters - a std::tuple of the remaining native parameters to transcribe - * @tparam WasmParameters - a std::tuple of the transribed parameters - */ -template -struct intrinsic_invoker_impl; - -/** - * Specialization for the fully transcribed signature - * @tparam Ret - the return type of the native function - */ -template -struct intrinsic_invoker_impl> { - using next_method_type = Ret (*)(wabt_apply_instance_vars&, const TypedValues&, int); - - template - static TypedValue invoke(wabt_apply_instance_vars& vars, const TypedValues& args) { - return convert_native_to_literal(vars, Method(vars, args, args.size() - 1)); - } - - template - static const auto fn() { - return invoke; - } -}; - -/** - * specialization of the fully transcribed signature for void return values - * @tparam Translated - the arguments to the wasm function - */ -template<> -struct intrinsic_invoker_impl> { - using next_method_type = void_type (*)(wabt_apply_instance_vars&, const TypedValues&, int); - - template - static TypedValue invoke(wabt_apply_instance_vars& vars, const TypedValues& args) { - Method(vars, args, args.size() - 1); - return TypedValue(Type::Void); - } - - template - static const auto fn() { - return invoke; - } -}; - -/** - * Sepcialization for transcribing a simple type in the native method signature - * @tparam Ret - the return type of the native method - * @tparam Input - the type of the native parameter to transcribe - * @tparam Inputs - the remaining native parameters to transcribe - */ -template -struct intrinsic_invoker_impl> { - using next_step = intrinsic_invoker_impl>; - using then_type = Ret (*)(wabt_apply_instance_vars&, Input, Inputs..., const TypedValues&, int); - - template - static Ret translate_one(wabt_apply_instance_vars& vars, Inputs... rest, const TypedValues& args, int offset) { - auto& last = args.at(offset); - auto native = convert_literal_to_native(last); - return Then(vars, native, rest..., args, (uint32_t)offset - 1); - }; - - template - static const auto fn() { - return next_step::template fn>(); - } -}; - -/** - * Specialization for transcribing a array_ptr type in the native method signature - * This type transcribes into 2 wasm parameters: a pointer and byte length and checks the validity of that memory - * range before dispatching to the native method - * - * @tparam Ret - the return type of the native method - * @tparam Inputs - the remaining native parameters to transcribe - */ -template -struct intrinsic_invoker_impl, uint32_t, Inputs...>> { - using next_step = intrinsic_invoker_impl>; - using then_type = Ret(*)(wabt_apply_instance_vars&, array_ptr, uint32_t, Inputs..., const TypedValues&, int); - - template - static auto translate_one(wabt_apply_instance_vars& vars, Inputs... rest, const TypedValues& args, int offset) -> std::enable_if_t::value, Ret> { - static_assert(!std::is_pointer::value, "Currently don't support array of pointers"); - uint32_t ptr = args.at((uint32_t)offset - 1).get_i32(); - size_t length = args.at((uint32_t)offset).get_i32(); - T* base = array_ptr_impl(vars, ptr, length); - if ( reinterpret_cast(base) % alignof(T) != 0 ) { - if(vars.ctx.control.contracts_console()) - wlog( "misaligned array of const values" ); - std::vector > copy(length > 0 ? length : 1); - T* copy_ptr = ©[0]; - memcpy( (void*)copy_ptr, (void*)base, length * sizeof(T) ); - return Then(vars, static_cast>(copy_ptr), length, rest..., args, (uint32_t)offset - 2); - } - return Then(vars, static_cast>(base), length, rest..., args, (uint32_t)offset - 2); - }; - - template - static auto translate_one(wabt_apply_instance_vars& vars, Inputs... rest, const TypedValues& args, int offset) -> std::enable_if_t::value, Ret> { - static_assert(!std::is_pointer::value, "Currently don't support array of pointers"); - uint32_t ptr = args.at((uint32_t)offset - 1).get_i32(); - size_t length = args.at((uint32_t)offset).get_i32(); - T* base = array_ptr_impl(vars, ptr, length); - if ( reinterpret_cast(base) % alignof(T) != 0 ) { - if(vars.ctx.control.contracts_console()) - wlog( "misaligned array of values" ); - std::vector > copy(length > 0 ? length : 1); - T* copy_ptr = ©[0]; - memcpy( (void*)copy_ptr, (void*)base, length * sizeof(T) ); - Ret ret = Then(vars, static_cast>(copy_ptr), length, rest..., args, (uint32_t)offset - 2); - memcpy( (void*)base, (void*)copy_ptr, length * sizeof(T) ); - return ret; - } - return Then(vars, static_cast>(base), length, rest..., args, (uint32_t)offset - 2); - }; - - template - static const auto fn() { - return next_step::template fn>(); - } -}; - -/** - * Specialization for transcribing a null_terminated_ptr type in the native method signature - * This type transcribes 1 wasm parameters: a char pointer which is validated to contain - * a null value before the end of the allocated memory. - * - * @tparam Ret - the return type of the native method - * @tparam Inputs - the remaining native parameters to transcribe - */ -template -struct intrinsic_invoker_impl> { - using next_step = intrinsic_invoker_impl>; - using then_type = Ret(*)(wabt_apply_instance_vars&, null_terminated_ptr, Inputs..., const TypedValues&, int); - - template - static Ret translate_one(wabt_apply_instance_vars& vars, Inputs... rest, const TypedValues& args, int offset) { - uint32_t ptr = args.at((uint32_t)offset).get_i32(); - return Then(vars, null_terminated_ptr_impl(vars, ptr), rest..., args, (uint32_t)offset - 1); - }; - - template - static const auto fn() { - return next_step::template fn>(); - } -}; - -/** - * Specialization for transcribing a pair of array_ptr types in the native method signature that share size - * This type transcribes into 3 wasm parameters: 2 pointers and byte length and checks the validity of those memory - * ranges before dispatching to the native method - * - * @tparam Ret - the return type of the native method - * @tparam Inputs - the remaining native parameters to transcribe - */ -template -struct intrinsic_invoker_impl, array_ptr, uint32_t, Inputs...>> { - using next_step = intrinsic_invoker_impl>; - using then_type = Ret(*)(wabt_apply_instance_vars&, array_ptr, array_ptr, uint32_t, Inputs..., const TypedValues&, int); - - template - static Ret translate_one(wabt_apply_instance_vars& vars, Inputs... rest, const TypedValues& args, int offset) { - uint32_t ptr_t = args.at((uint32_t)offset - 2).get_i32(); - uint32_t ptr_u = args.at((uint32_t)offset - 1).get_i32(); - size_t length = args.at((uint32_t)offset).get_i32(); - static_assert(std::is_same, char>::value && std::is_same, char>::value, "Currently only support array of (const)chars"); - return Then(vars, array_ptr_impl(vars, ptr_t, length), array_ptr_impl(vars, ptr_u, length), length, args, (uint32_t)offset - 3); - }; - - template - static const auto fn() { - return next_step::template fn>(); - } -}; - -/** - * Specialization for transcribing memset parameters - * - * @tparam Ret - the return type of the native method - * @tparam Inputs - the remaining native parameters to transcribe - */ -template -struct intrinsic_invoker_impl, int, uint32_t>> { - using next_step = intrinsic_invoker_impl>; - using then_type = Ret(*)(wabt_apply_instance_vars&, array_ptr, int, uint32_t, const TypedValues&, int); - - template - static Ret translate_one(wabt_apply_instance_vars& vars, const TypedValues& args, int offset) { - uint32_t ptr = args.at((uint32_t)offset - 2).get_i32(); - uint32_t value = args.at((uint32_t)offset - 1).get_i32(); - size_t length = args.at((uint32_t)offset).get_i32(); - return Then(vars, array_ptr_impl(vars, ptr, length), value, length, args, (uint32_t)offset - 3); - }; - - template - static const auto fn() { - return next_step::template fn>(); - } -}; - -/** - * Specialization for transcribing a pointer type in the native method signature - * This type transcribes into an int32 pointer checks the validity of that memory - * range before dispatching to the native method - * - * @tparam Ret - the return type of the native method - * @tparam Inputs - the remaining native parameters to transcribe - */ -template -struct intrinsic_invoker_impl> { - using next_step = intrinsic_invoker_impl>; - using then_type = Ret (*)(wabt_apply_instance_vars&, T *, Inputs..., const TypedValues&, int); - - template - static auto translate_one(wabt_apply_instance_vars& vars, Inputs... rest, const TypedValues& args, int offset) -> std::enable_if_t::value, Ret> { - uint32_t ptr = args.at((uint32_t)offset).get_i32(); - T* base = array_ptr_impl(vars, ptr, 1); - if ( reinterpret_cast(base) % alignof(T) != 0 ) { - if(vars.ctx.control.contracts_console()) - wlog( "misaligned const pointer" ); - std::remove_const_t copy; - T* copy_ptr = © - memcpy( (void*)copy_ptr, (void*)base, sizeof(T) ); - return Then(vars, copy_ptr, rest..., args, (uint32_t)offset - 1); - } - return Then(vars, base, rest..., args, (uint32_t)offset - 1); - }; - - template - static auto translate_one(wabt_apply_instance_vars& vars, Inputs... rest, const TypedValues& args, int offset) -> std::enable_if_t::value, Ret> { - uint32_t ptr = args.at((uint32_t)offset).get_i32(); - T* base = array_ptr_impl(vars, ptr, 1); - if ( reinterpret_cast(base) % alignof(T) != 0 ) { - if(vars.ctx.control.contracts_console()) - wlog( "misaligned pointer" ); - T copy; - memcpy( (void*)©, (void*)base, sizeof(T) ); - Ret ret = Then(vars, ©, rest..., args, (uint32_t)offset - 1); - memcpy( (void*)base, (void*)©, sizeof(T) ); - return ret; - } - return Then(vars, base, rest..., args, (uint32_t)offset - 1); - }; - - template - static const auto fn() { - return next_step::template fn>(); - } -}; - -/** - * Specialization for transcribing a reference to a name which can be passed as a native value - * This type transcribes into a native type which is loaded by value into a - * variable on the stack and then passed by reference to the intrinsic. - * - * @tparam Ret - the return type of the native method - * @tparam Inputs - the remaining native parameters to transcribe - */ -template -struct intrinsic_invoker_impl> { - using next_step = intrinsic_invoker_impl>; - using then_type = Ret (*)(wabt_apply_instance_vars&, const name&, Inputs..., const TypedValues&, int); - - template - static Ret translate_one(wabt_apply_instance_vars& vars, Inputs... rest, const TypedValues& args, int offset) { - uint64_t wasm_value = args.at((uint32_t)offset).get_i64(); - auto value = name(wasm_value); - return Then(vars, value, rest..., args, (uint32_t)offset - 1); - } - - template - static const auto fn() { - return next_step::template fn>(); - } -}; - -/** - * Specialization for transcribing a reference to a fc::time_point_sec which can be passed as a native value - * This type transcribes into a native type which is loaded by value into a - * variable on the stack and then passed by reference to the intrinsic. - * - * @tparam Ret - the return type of the native method - * @tparam Inputs - the remaining native parameters to transcribe - */ -template -struct intrinsic_invoker_impl> { - using next_step = intrinsic_invoker_impl>; - using then_type = Ret (*)(wabt_apply_instance_vars&, const fc::time_point_sec&, Inputs..., const TypedValues&, int); - - template - static Ret translate_one(wabt_apply_instance_vars& vars, Inputs... rest, const TypedValues& args, int offset) { - uint32_t wasm_value = args.at((uint32_t)offset).get_i32(); - auto value = fc::time_point_sec(wasm_value); - return Then(vars, value, rest..., args, (uint32_t)offset - 1); - } - - template - static const auto fn() { - return next_step::template fn>(); - } -}; - - -/** - * Specialization for transcribing a reference type in the native method signature - * This type transcribes into an int32 pointer checks the validity of that memory - * range before dispatching to the native method - * - * @tparam Ret - the return type of the native method - * @tparam Inputs - the remaining native parameters to transcribe - */ -template -struct intrinsic_invoker_impl> { - using next_step = intrinsic_invoker_impl>; - using then_type = Ret (*)(wabt_apply_instance_vars&, T &, Inputs..., const TypedValues&, int); - - template - static auto translate_one(wabt_apply_instance_vars& vars, Inputs... rest, const TypedValues& args, int offset) -> std::enable_if_t::value, Ret> { - // references cannot be created for null pointers - uint32_t ptr = args.at((uint32_t)offset).get_i32(); - EOS_ASSERT(ptr != 0, binaryen_exception, "references cannot be created for null pointers"); - T* base = array_ptr_impl(vars, ptr, 1); - if ( reinterpret_cast(base) % alignof(T) != 0 ) { - if(vars.ctx.control.contracts_console()) - wlog( "misaligned const reference" ); - std::remove_const_t copy; - T* copy_ptr = © - memcpy( (void*)copy_ptr, (void*)base, sizeof(T) ); - return Then(vars, *copy_ptr, rest..., args, (uint32_t)offset - 1); - } - return Then(vars, *base, rest..., args, (uint32_t)offset - 1); - } - - template - static auto translate_one(wabt_apply_instance_vars& vars, Inputs... rest, const TypedValues& args, int offset) -> std::enable_if_t::value, Ret> { - // references cannot be created for null pointers - uint32_t ptr = args.at((uint32_t)offset).get_i32(); - EOS_ASSERT(ptr != 0, binaryen_exception, "references cannot be created for null pointers"); - T* base = array_ptr_impl(vars, ptr, 1); - if ( reinterpret_cast(base) % alignof(T) != 0 ) { - if(vars.ctx.control.contracts_console()) - wlog( "misaligned reference" ); - T copy; - memcpy( (void*)©, (void*)base, sizeof(T) ); - Ret ret = Then(vars, copy, rest..., args, (uint32_t)offset - 1); - memcpy( (void*)base, (void*)©, sizeof(T) ); - return ret; - } - return Then(vars, *base, rest..., args, (uint32_t)offset - 1); - } - - - template - static const auto fn() { - return next_step::template fn>(); - } -}; - -extern apply_context* fixme_context; - -/** - * forward declaration of a wrapper class to call methods of the class - */ -template -struct intrinsic_function_invoker { - using impl = intrinsic_invoker_impl>; - - template - static Ret wrapper(wabt_apply_instance_vars& vars, Params... params, const TypedValues&, int) { - class_from_wasm::value(vars.ctx).checktime(); - return (class_from_wasm::value(vars.ctx).*Method)(params...); - } - - template - static const intrinsic_registrator::intrinsic_fn fn() { - return impl::template fn>(); - } -}; - -template -struct intrinsic_function_invoker { - using impl = intrinsic_invoker_impl>; - - template - static void_type wrapper(wabt_apply_instance_vars& vars, Params... params, const TypedValues& args, int offset) { - class_from_wasm::value(vars.ctx).checktime(); - (class_from_wasm::value(vars.ctx).*Method)(params...); - return void_type(); - } - - template - static const intrinsic_registrator::intrinsic_fn fn() { - return impl::template fn>(); - } - -}; - -template -struct intrinsic_function_invoker_wrapper; - -template -struct void_ret_wrapper { - using type = T; -}; - -template<> -struct void_ret_wrapper { - using type = char; -}; - -template -using void_ret_wrapper_t = typename void_ret_wrapper::type; - -template -struct intrinsic_function_invoker_wrapper { - static_assert( !(std::is_pointer_v && alignof(std::remove_pointer_t>) != 1) && - !(std::is_lvalue_reference_v && alignof(std::remove_reference_t>) != 1), - "intrinsics should only return a reference or pointer with single byte alignment"); - using type = intrinsic_function_invoker; -}; - -template -struct intrinsic_function_invoker_wrapper { - static_assert( !(std::is_pointer_v && alignof(std::remove_pointer_t>) != 1) && - !(std::is_lvalue_reference_v && alignof(std::remove_reference_t>) != 1), - "intrinsics should only return a reference or pointer with single byte alignment"); - using type = intrinsic_function_invoker; -}; - -template -struct intrinsic_function_invoker_wrapper { - static_assert( !(std::is_pointer_v && alignof(std::remove_pointer_t>) != 1) && - !(std::is_lvalue_reference_v && alignof(std::remove_reference_t>) != 1), - "intrinsics should only return a reference or pointer with single byte alignment"); - using type = intrinsic_function_invoker; -}; - -template -struct intrinsic_function_invoker_wrapper { - static_assert( !(std::is_pointer_v && alignof(std::remove_pointer_t>) != 1) && - !(std::is_lvalue_reference_v && alignof(std::remove_reference_t>) != 1), - "intrinsics should only return a reference or pointer with single byte alignment"); - using type = intrinsic_function_invoker; -}; - -#define __INTRINSIC_NAME(LABEL, SUFFIX) LABEL##SUFFIX -#define _INTRINSIC_NAME(LABEL, SUFFIX) __INTRINSIC_NAME(LABEL,SUFFIX) - -#define _REGISTER_WABT_INTRINSIC(CLS, MOD, METHOD, WASM_SIG, NAME, SIG)\ - static eosio::chain::webassembly::wabt_runtime::intrinsic_registrator _INTRINSIC_NAME(__wabt_intrinsic_fn, __COUNTER__) (\ - MOD,\ - NAME,\ - eosio::chain::webassembly::wabt_runtime::wabt_function_type_provider::type(),\ - eosio::chain::webassembly::wabt_runtime::intrinsic_function_invoker_wrapper::type::fn<&CLS::METHOD>()\ - );\ - -} } } }// eosio::chain::webassembly::wabt_runtime diff --git a/libraries/chain/include/eosio/chain/webassembly/wavm.hpp b/libraries/chain/include/eosio/chain/webassembly/wavm.hpp deleted file mode 100644 index 7dde4a443bd..00000000000 --- a/libraries/chain/include/eosio/chain/webassembly/wavm.hpp +++ /dev/null @@ -1,155 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include "Runtime/Runtime.h" -#include "IR/Types.h" - - -namespace eosio { namespace chain { namespace webassembly { namespace wavm { - -using namespace IR; -using namespace Runtime; -using namespace fc; -using namespace eosio::chain::webassembly::common; - -class wavm_runtime : public eosio::chain::wasm_runtime_interface { - public: - wavm_runtime(); - ~wavm_runtime(); - bool inject_module(IR::Module&) override; - std::unique_ptr instantiate_module(const char* code_bytes, size_t code_size, std::vector initial_memory, - const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override; - - void immediately_exit_currently_running_module() override; -}; - -//This is a temporary hack for the single threaded implementation -struct running_instance_context { - MemoryInstance* memory; - apply_context* apply_ctx; -}; -extern running_instance_context the_running_instance_context; - -template -struct wasm_to_value_type; - -template<> -struct wasm_to_value_type { - static constexpr auto value = ValueType::f32; -}; - -template<> -struct wasm_to_value_type { - static constexpr auto value = ValueType::f64; -}; -template<> -struct wasm_to_value_type { - static constexpr auto value = ValueType::i32; -}; -template<> -struct wasm_to_value_type { - static constexpr auto value = ValueType::i64; -}; - -template -constexpr auto wasm_to_value_type_v = wasm_to_value_type::value; - -template -struct wasm_to_rvalue_type; -template<> -struct wasm_to_rvalue_type { - static constexpr auto value = ResultType::f32; -}; -template<> -struct wasm_to_rvalue_type { - static constexpr auto value = ResultType::f64; -}; -template<> -struct wasm_to_rvalue_type { - static constexpr auto value = ResultType::i32; -}; -template<> -struct wasm_to_rvalue_type { - static constexpr auto value = ResultType::i64; -}; -template<> -struct wasm_to_rvalue_type { - static constexpr auto value = ResultType::none; -}; -template<> -struct wasm_to_rvalue_type { - static constexpr auto value = ResultType::i64; -}; -template<> -struct wasm_to_rvalue_type { - static constexpr auto value = ResultType::i64; -}; - -template<> -struct wasm_to_rvalue_type { - static constexpr auto value = ResultType::i32; -}; - -template<> -struct wasm_to_rvalue_type { - static constexpr auto value = ResultType::i32; -}; - - -template -constexpr auto wasm_to_rvalue_type_v = wasm_to_rvalue_type::value; - -template -struct is_reference_from_value { - static constexpr bool value = false; -}; - -template<> -struct is_reference_from_value { - static constexpr bool value = true; -}; - -template<> -struct is_reference_from_value { - static constexpr bool value = true; -}; - -template -constexpr bool is_reference_from_value_v = is_reference_from_value::value; - - - -struct void_type { -}; - -/** - * Forward declaration of provider for FunctionType given a desired C ABI signature - */ -template -struct wasm_function_type_provider; - -/** - * specialization to destructure return and arguments - */ -template -struct wasm_function_type_provider { - static const FunctionType *type() { - return FunctionType::get(wasm_to_rvalue_type_v, {wasm_to_value_type_v ...}); - } -}; - -#define __INTRINSIC_NAME(LABEL, SUFFIX) LABEL##SUFFIX -#define _INTRINSIC_NAME(LABEL, SUFFIX) __INTRINSIC_NAME(LABEL,SUFFIX) - -#define _REGISTER_WAVM_INTRINSIC(CLS, MOD, METHOD, WASM_SIG, NAME, SIG)\ - static Intrinsics::Function _INTRINSIC_NAME(__intrinsic_fn, __COUNTER__) (\ - MOD "." NAME,\ - eosio::chain::webassembly::wavm::wasm_function_type_provider::type(),\ - nullptr\ - ); - -} } } }// eosio::chain::webassembly::wavm diff --git a/libraries/chain/include/eosio/chain/whitelisted_intrinsics.hpp b/libraries/chain/include/eosio/chain/whitelisted_intrinsics.hpp index 3997993d77c..81b80be52ae 100644 --- a/libraries/chain/include/eosio/chain/whitelisted_intrinsics.hpp +++ b/libraries/chain/include/eosio/chain/whitelisted_intrinsics.hpp @@ -6,13 +6,11 @@ namespace eosio { namespace chain { using whitelisted_intrinsics_type = shared_flat_multimap; - // TODO: Improve performance by using std::string_view when we switch to C++17. + bool is_intrinsic_whitelisted( const whitelisted_intrinsics_type& whitelisted_intrinsics, std::string_view name ); - bool is_intrinsic_whitelisted( const whitelisted_intrinsics_type& whitelisted_intrinsics, const std::string& name ); + void add_intrinsic_to_whitelist( whitelisted_intrinsics_type& whitelisted_intrinsics, std::string_view name ); - void add_intrinsic_to_whitelist( whitelisted_intrinsics_type& whitelisted_intrinsics, const std::string& name ); - - void remove_intrinsic_from_whitelist( whitelisted_intrinsics_type& whitelisted_intrinsics, const std::string& name ); + void remove_intrinsic_from_whitelist( whitelisted_intrinsics_type& whitelisted_intrinsics, std::string_view name ); void reset_intrinsic_whitelist( whitelisted_intrinsics_type& whitelisted_intrinsics, const std::set& s ); diff --git a/libraries/chain/merkle.cpp b/libraries/chain/merkle.cpp index 9c6ea420981..e4211f7bfd6 100644 --- a/libraries/chain/merkle.cpp +++ b/libraries/chain/merkle.cpp @@ -32,7 +32,7 @@ bool is_canonical_right(const digest_type& val) { } -digest_type merkle(vector ids) { +digest_type merkle(deque ids) { if( 0 == ids.size() ) { return digest_type(); } while( ids.size() > 1 ) { diff --git a/libraries/chain/name.cpp b/libraries/chain/name.cpp index 79ffe1603d4..6d35b562e6d 100644 --- a/libraries/chain/name.cpp +++ b/libraries/chain/name.cpp @@ -1,18 +1,11 @@ #include #include #include -#include -#include namespace eosio::chain { void name::set( std::string_view str ) { - const auto len = str.size(); - EOS_ASSERT(len <= 13, name_type_exception, "Name is longer than 13 characters (${name}) ", ("name", std::string(str))); value = string_to_uint64_t(str); - EOS_ASSERT(to_string() == str, name_type_exception, - "Name not properly normalized (name: ${name}, normalized: ${normalized}) ", - ("name", std::string(str))("normalized", to_string())); } // keep in sync with name::to_string() in contract definition for name @@ -32,6 +25,32 @@ namespace eosio::chain { return str; } + bool is_string_valid_name(std::string_view str) + { + size_t slen = str.size(); + if( slen > 13) + return false; + + size_t len = (slen <= 12) ? slen : 12; + for( size_t i = 0; i < len; ++i ) { + char c = str[i]; + if ((c >= 'a' && c <= 'z') || (c >= '1' && c <= '5') || (c == '.')) + continue; + else + return false; + } + + if( slen == 13) { + char c = str[12]; + if ((c >= 'a' && c <= 'j') || (c >= '1' && c <= '5') || (c == '.')) + return true; + else + return false; + } + + return true; + } + } // eosio::chain namespace fc { diff --git a/libraries/chain/producer_schedule.cpp b/libraries/chain/producer_schedule.cpp index 23aa205273f..b4c13338d53 100644 --- a/libraries/chain/producer_schedule.cpp +++ b/libraries/chain/producer_schedule.cpp @@ -3,7 +3,7 @@ namespace eosio { namespace chain { fc::variant producer_authority::get_abi_variant() const { - auto authority_variant = authority.visit([](const auto& a){ + auto authority_variant = std::visit([](const auto& a){ fc::variant value; fc::to_variant(a, value); @@ -13,11 +13,11 @@ fc::variant producer_authority::get_abi_variant() const { std::move(type), std::move(value) }; - }); + }, authority); return fc::mutable_variant_object() ("producer_name", producer_name) ("authority", std::move(authority_variant)); } -} } /// eosio::chain \ No newline at end of file +} } /// eosio::chain diff --git a/libraries/chain/protocol_feature_manager.cpp b/libraries/chain/protocol_feature_manager.cpp index d61a7fe25c4..82da9deb542 100644 --- a/libraries/chain/protocol_feature_manager.cpp +++ b/libraries/chain/protocol_feature_manager.cpp @@ -186,6 +186,50 @@ may use a new `set_proposed_producers_ex` intrinsic to access extended features. */ {} } ) + ( builtin_protocol_feature_t::action_return_value, builtin_protocol_feature_spec{ + "ACTION_RETURN_VALUE", + fc::variant("69b064c5178e2738e144ed6caa9349a3995370d78db29e494b3126ebd9111966").as(), + // SHA256 hash of the raw message below within the comment delimiters (do not modify message below). +/* +Builtin protocol feature: ACTION_RETURN_VALUE + +Enables new `set_action_return_value` intrinsic which sets a value that is included in action_receipt. +*/ + {} + } ) + ( builtin_protocol_feature_t::kv_database, builtin_protocol_feature_spec{ + "KV_DATABASE", + fc::variant("14cfb3252a5fa3ae4c764929e0bbc467528990c9cc46aefcc7f16367f28b6278").as(), + // SHA256 hash of the raw message below within the comment delimiters (do not modify message below). +/* +Builtin protocol feature: KV_DATABASE + +Enables usage of key-value database intrinsics. +*/ + {} + } ) + ( builtin_protocol_feature_t::configurable_wasm_limits, builtin_protocol_feature_spec{ + "CONFIGURABLE_WASM_LIMITS", + fc::variant("67f5f1e92cbf6f7276e7b3fc8c2ad23e63448e657641a1e5de69bccd114542d6").as(), + // SHA256 hash of the raw message below within the comment delimiters (do not modify message below). +/* +Builtin protocol feature: CONFIGURABLE_WASM_LIMITS + +Allows privileged contracts to set the constraints on WebAssembly code. +*/ + {} + } ) + ( builtin_protocol_feature_t::blockchain_parameters, builtin_protocol_feature_spec{ + "BLOCKCHAIN_PARAMETERS", + fc::variant("70787548dcea1a2c52c913a37f74ce99e6caae79110d7ca7b859936a0075b314").as(), + {} + } ) + // SHA256 hash of the raw message below within the comment delimiters (do not modify message below). +/* +Builtin protocol feature: BLOCKCHAIN_PARAMETERS + +Allows privileged contracts to get and set subsets of blockchain parameters. +*/ ; @@ -344,7 +388,7 @@ may use a new `set_proposed_producers_ex` intrinsic to access extended features. return recognized_t::ready; } - optional protocol_feature_set::get_builtin_digest( builtin_protocol_feature_t feature_codename )const { + std::optional protocol_feature_set::get_builtin_digest( builtin_protocol_feature_t feature_codename )const { uint32_t indx = static_cast( feature_codename ); if( indx >= _recognized_builtin_protocol_features.size() ) @@ -492,8 +536,10 @@ may use a new `set_proposed_producers_ex` intrinsic to access extended features. - protocol_feature_manager::protocol_feature_manager( protocol_feature_set&& pfs ) - :_protocol_feature_set( std::move(pfs) ) + protocol_feature_manager::protocol_feature_manager( + protocol_feature_set&& pfs, + std::function get_deep_mind_logger + ):_protocol_feature_set( std::move(pfs) ), _get_deep_mind_logger(get_deep_mind_logger) { _builtin_protocol_features.resize( _protocol_feature_set._recognized_builtin_protocol_features.size() ); } @@ -673,6 +719,13 @@ may use a new `set_proposed_producers_ex` intrinsic to access extended features. ("digest", feature_digest) ); + if (auto dm_logger = _get_deep_mind_logger()) { + fc_dlog(*dm_logger, "FEATURE_OP ACTIVATE ${feature_digest} ${feature}", + ("feature_digest", feature_digest) + ("feature", itr->to_variant()) + ); + } + _activated_protocol_features.push_back( protocol_feature_entry{itr, current_block_num} ); _builtin_protocol_features[indx].previous = _head_of_builtin_activation_list; _builtin_protocol_features[indx].activation_block_num = current_block_num; diff --git a/libraries/chain/resource_limits.cpp b/libraries/chain/resource_limits.cpp index d834ac42420..d8d734b986b 100644 --- a/libraries/chain/resource_limits.cpp +++ b/libraries/chain/resource_limits.cpp @@ -52,16 +52,28 @@ void resource_limits_manager::add_indices() { } void resource_limits_manager::initialize_database() { - const auto& config = _db.create([](resource_limits_config_object& config){ + const auto& config = _db.create([this](resource_limits_config_object& config){ // see default settings in the declaration + + if (auto dm_logger = _get_deep_mind_logger()) { + fc_dlog(*dm_logger, "RLIMIT_OP CONFIG INS ${data}", + ("data", config) + ); + } }); - _db.create([&config](resource_limits_state_object& state){ + _db.create([this, &config](resource_limits_state_object& state){ // see default settings in the declaration // start the chain off in a way that it is "congested" aka slow-start state.virtual_cpu_limit = config.cpu_limit_parameters.max; state.virtual_net_limit = config.net_limit_parameters.max; + + if (auto dm_logger = _get_deep_mind_logger()) { + fc_dlog(*dm_logger, "RLIMIT_OP STATE INS ${data}", + ("data", state) + ); + } }); } @@ -75,8 +87,25 @@ void resource_limits_manager::add_to_snapshot( const snapshot_writer_ptr& snapsh }); } -void resource_limits_manager::read_from_snapshot( const snapshot_reader_ptr& snapshot ) { - resource_index_set::walk_indices([this, &snapshot]( auto utils ){ +void resource_limits_manager::read_from_snapshot( const snapshot_reader_ptr& snapshot, uint32_t version ) { + resource_index_set::walk_indices([this, &snapshot, version]( auto utils ){ + using value_t = typename decltype(utils)::index_t::value_type; + if constexpr ( !std::is_same_v ) { + using v3 = typename value_t::v3; + if ( v3::minimum_version <= version && version <= v3::maximum_version ) { + snapshot->read_section([this]( auto& section ) { + bool more = !section.empty(); + while(more) { + decltype(utils)::create(_db, [this, §ion, &more]( auto &row ) { + v3 legacy_row; + more = section.read_row(legacy_row, _db); + row.initialize_from(legacy_row); + }); + } + }); + return; + } + } snapshot->read_section([this]( auto& section ) { bool more = !section.empty(); while(more) { @@ -91,10 +120,22 @@ void resource_limits_manager::read_from_snapshot( const snapshot_reader_ptr& sna void resource_limits_manager::initialize_account(const account_name& account) { _db.create([&]( resource_limits_object& bl ) { bl.owner = account; + + if (auto dm_logger = _get_deep_mind_logger()) { + fc_dlog(*dm_logger, "RLIMIT_OP ACCOUNT_LIMITS INS ${data}", + ("data", bl) + ); + } }); _db.create([&]( resource_usage_object& bu ) { bu.owner = account; + + if (auto dm_logger = _get_deep_mind_logger()) { + fc_dlog(*dm_logger, "RLIMIT_OP ACCOUNT_USAGE INS ${data}", + ("data", bu) + ); + } }); } @@ -104,9 +145,16 @@ void resource_limits_manager::set_block_parameters(const elastic_limit_parameter const auto& config = _db.get(); if( config.cpu_limit_parameters == cpu_limit_parameters && config.net_limit_parameters == net_limit_parameters ) return; + _db.modify(config, [&](resource_limits_config_object& c){ c.cpu_limit_parameters = cpu_limit_parameters; c.net_limit_parameters = net_limit_parameters; + + if (auto dm_logger = _get_deep_mind_logger()) { + fc_dlog(*dm_logger, "RLIMIT_OP CONFIG UPD ${data}", + ("data", c) + ); + } }); } @@ -136,6 +184,12 @@ void resource_limits_manager::add_transaction_usage(const flat_set _db.modify( usage, [&]( auto& bu ){ bu.net_usage.add( net_usage, time_slot, config.account_net_usage_average_window ); bu.cpu_usage.add( cpu_usage, time_slot, config.account_cpu_usage_average_window ); + + if (auto dm_logger = _get_deep_mind_logger()) { + fc_dlog(*dm_logger, "RLIMIT_OP ACCOUNT_USAGE UPD ${data}", + ("data", bu) + ); + } }); if( cpu_weight >= 0 && state.total_cpu_weight > 0 ) { @@ -187,7 +241,7 @@ void resource_limits_manager::add_transaction_usage(const flat_set EOS_ASSERT( state.pending_net_usage <= config.net_limit_parameters.max, block_resource_exhausted, "Block has insufficient net resources" ); } -void resource_limits_manager::add_pending_ram_usage( const account_name account, int64_t ram_delta ) { +void resource_limits_manager::add_pending_ram_usage( const account_name account, int64_t ram_delta, const storage_usage_trace& trace ) { if (ram_delta == 0) { return; } @@ -200,7 +254,20 @@ void resource_limits_manager::add_pending_ram_usage( const account_name account, "Ram usage delta would underflow UINT64_MAX"); _db.modify( usage, [&]( auto& u ) { - u.ram_usage += ram_delta; + u.ram_usage += ram_delta; + + if (auto dm_logger = _get_deep_mind_logger()) { + fc_dlog(*dm_logger, "RAM_OP ${action_id} ${event_id} ${family} ${operation} ${legacy_tag} ${payer} ${new_usage} ${delta}", + ("action_id", trace.action_id) + ("event_id", trace.event_id) + ("family", trace.family) + ("operation", trace.operation) + ("legacy_tag", trace.legacy_tag) + ("payer", account) + ("new_usage", u.ram_usage) + ("delta", ram_delta) + ); + } }); } @@ -220,75 +287,73 @@ int64_t resource_limits_manager::get_account_ram_usage( const account_name& name return _db.get( name ).ram_usage; } - -bool resource_limits_manager::set_account_limits( const account_name& account, int64_t ram_bytes, int64_t net_weight, int64_t cpu_weight) { - //const auto& usage = _db.get( account ); +const resource_limits_object& resource_limits_manager::get_or_create_pending_account_limits( const account_name& account ) { /* * Since we need to delay these until the next resource limiting boundary, these are created in a "pending" * state or adjusted in an existing "pending" state. The chain controller will collapse "pending" state into * the actual state at the next appropriate boundary. */ - auto find_or_create_pending_limits = [&]() -> const resource_limits_object& { - const auto* pending_limits = _db.find( boost::make_tuple(true, account) ); - if (pending_limits == nullptr) { - const auto& limits = _db.get( boost::make_tuple(false, account)); - return _db.create([&](resource_limits_object& pending_limits){ - pending_limits.owner = limits.owner; - pending_limits.ram_bytes = limits.ram_bytes; - pending_limits.net_weight = limits.net_weight; - pending_limits.cpu_weight = limits.cpu_weight; - pending_limits.pending = true; - }); - } else { - return *pending_limits; - } - }; + const auto* pending_limits = _db.find( boost::make_tuple(true, account) ); + if (pending_limits == nullptr) { + const auto& limits = _db.get( boost::make_tuple(false, account)); + return _db.create([&](resource_limits_object& pending_limits){ + pending_limits.owner = limits.owner; + pending_limits.ram_bytes = limits.ram_bytes; + pending_limits.net_weight = limits.net_weight; + pending_limits.cpu_weight = limits.cpu_weight; + pending_limits.pending = true; + }); + } else { + return *pending_limits; + } +} +bool resource_limits_manager::set_account_limits( const account_name& account, int64_t ram_bytes, int64_t net_weight, int64_t cpu_weight) { // update the users weights directly - auto& limits = find_or_create_pending_limits(); + auto& limits = get_or_create_pending_account_limits( account ); bool decreased_limit = false; if( ram_bytes >= 0 ) { - decreased_limit = ( (limits.ram_bytes < 0) || (ram_bytes < limits.ram_bytes) ); - - /* - if( limits.ram_bytes < 0 ) { - EOS_ASSERT(ram_bytes >= usage.ram_usage, wasm_execution_error, "converting unlimited account would result in overcommitment [commit=${c}, desired limit=${l}]", ("c", usage.ram_usage)("l", ram_bytes)); - } else { - EOS_ASSERT(ram_bytes >= usage.ram_usage, wasm_execution_error, "attempting to release committed ram resources [commit=${c}, desired limit=${l}]", ("c", usage.ram_usage)("l", ram_bytes)); - } - */ } _db.modify( limits, [&]( resource_limits_object& pending_limits ){ pending_limits.ram_bytes = ram_bytes; pending_limits.net_weight = net_weight; pending_limits.cpu_weight = cpu_weight; + + if (auto dm_logger = _get_deep_mind_logger()) { + fc_dlog(*dm_logger, "RLIMIT_OP ACCOUNT_LIMITS UPD ${data}", + ("data", pending_limits) + ); + } }); return decreased_limit; } -void resource_limits_manager::get_account_limits( const account_name& account, int64_t& ram_bytes, int64_t& net_weight, int64_t& cpu_weight ) const { +const resource_limits_object& resource_limits_manager::get_account_limits( const account_name& account ) const { const auto* pending_buo = _db.find( boost::make_tuple(true, account) ); if (pending_buo) { - ram_bytes = pending_buo->ram_bytes; - net_weight = pending_buo->net_weight; - cpu_weight = pending_buo->cpu_weight; + return *pending_buo; } else { const auto& buo = _db.get( boost::make_tuple( false, account ) ); - ram_bytes = buo.ram_bytes; - net_weight = buo.net_weight; - cpu_weight = buo.cpu_weight; + return buo; } } +void resource_limits_manager::get_account_limits( const account_name& account, int64_t& ram_bytes, int64_t& net_weight, int64_t& cpu_weight ) const { + const auto& buo = get_account_limits(account); + ram_bytes = buo.ram_bytes; + net_weight = buo.net_weight; + cpu_weight = buo.cpu_weight; +} + bool resource_limits_manager::is_unlimited_cpu( const account_name& account ) const { - const auto* buo = _db.find( boost::make_tuple(false, account) ); - if (buo) { - return buo->cpu_weight == -1; + const auto* rlo = _db.find( boost::make_tuple(false, account) ); + if (rlo) { + return rlo->cpu_weight == -1; } return false; } @@ -329,6 +394,12 @@ void resource_limits_manager::process_account_limit_updates() { multi_index.remove(*itr); } + + if (auto dm_logger = _get_deep_mind_logger()) { + fc_dlog(*dm_logger, "RLIMIT_OP STATE UPD ${data}", + ("data", state) + ); + } }); } @@ -346,6 +417,11 @@ void resource_limits_manager::process_block_usage(uint32_t block_num) { state.update_virtual_net_limit(config); state.pending_net_usage = 0; + if (auto dm_logger = _get_deep_mind_logger()) { + fc_dlog(*dm_logger, "RLIMIT_OP STATE UPD ${data}", + ("data", state) + ); + } }); } @@ -377,7 +453,8 @@ std::pair resource_limits_manager::get_account_cpu_limit( const a return {arl.available, greylisted}; } -std::pair resource_limits_manager::get_account_cpu_limit_ex( const account_name& name, uint32_t greylist_limit ) const { +std::pair +resource_limits_manager::get_account_cpu_limit_ex( const account_name& name, uint32_t greylist_limit, const std::optional& current_time) const { const auto& state = _db.get(); const auto& usage = _db.get(name); @@ -387,7 +464,7 @@ std::pair resource_limits_manager::get_account_cpu get_account_limits( name, x, y, cpu_weight ); if( cpu_weight < 0 || state.total_cpu_weight == 0 ) { - return {{ -1, -1, -1 }, false}; + return {{ -1, -1, -1, block_timestamp_type(usage.cpu_usage.last_ordinal), -1 }, false}; } account_resource_limit arl; @@ -421,6 +498,15 @@ std::pair resource_limits_manager::get_account_cpu arl.used = impl::downgrade_cast(cpu_used_in_window); arl.max = impl::downgrade_cast(max_user_use_in_window); + arl.last_usage_update_time = block_timestamp_type(usage.cpu_usage.last_ordinal); + arl.current_used = arl.used; + if ( current_time ) { + if (current_time->slot > usage.cpu_usage.last_ordinal) { + auto history_usage = usage.cpu_usage; + history_usage.add(0, current_time->slot, window_size); + arl.current_used = impl::downgrade_cast(impl::integer_divide_ceil((uint128_t)history_usage.value_ex * window_size, (uint128_t)config::rate_limiting_precision)); + } + } return {arl, greylisted}; } @@ -429,7 +515,8 @@ std::pair resource_limits_manager::get_account_net_limit( const a return {arl.available, greylisted}; } -std::pair resource_limits_manager::get_account_net_limit_ex( const account_name& name, uint32_t greylist_limit ) const { +std::pair +resource_limits_manager::get_account_net_limit_ex( const account_name& name, uint32_t greylist_limit, const std::optional& current_time) const { const auto& config = _db.get(); const auto& state = _db.get(); const auto& usage = _db.get(name); @@ -438,7 +525,7 @@ std::pair resource_limits_manager::get_account_net get_account_limits( name, x, net_weight, y ); if( net_weight < 0 || state.total_net_weight == 0) { - return {{ -1, -1, -1 }, false}; + return {{ -1, -1, -1, block_timestamp_type(usage.net_usage.last_ordinal), -1 }, false}; } account_resource_limit arl; @@ -472,6 +559,15 @@ std::pair resource_limits_manager::get_account_net arl.used = impl::downgrade_cast(net_used_in_window); arl.max = impl::downgrade_cast(max_user_use_in_window); + arl.last_usage_update_time = block_timestamp_type(usage.net_usage.last_ordinal); + arl.current_used = arl.used; + if ( current_time ) { + if (current_time->slot > usage.net_usage.last_ordinal) { + auto history_usage = usage.net_usage; + history_usage.add(0, current_time->slot, window_size); + arl.current_used = impl::downgrade_cast(impl::integer_divide_ceil((uint128_t)history_usage.value_ex * window_size, (uint128_t)config::rate_limiting_precision)); + } + } return {arl, greylisted}; } diff --git a/libraries/chain/trace.cpp b/libraries/chain/trace.cpp index 7ce555d8a11..8c21ee92ff4 100644 --- a/libraries/chain/trace.cpp +++ b/libraries/chain/trace.cpp @@ -1,3 +1,4 @@ +#include #include namespace eosio { namespace chain { diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index ee51d639304..4051ee0a0bd 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -133,21 +133,21 @@ signed_transaction::get_signature_keys( const chain_id_type& chain_id, fc::time_ return transaction::get_signature_keys(signatures, chain_id, deadline, context_free_data, recovered_pub_keys, allow_duplicate_keys); } -uint32_t packed_transaction::get_unprunable_size()const { +uint32_t packed_transaction_v0::get_unprunable_size()const { uint64_t size = config::fixed_net_overhead_of_packed_trx; size += packed_trx.size(); EOS_ASSERT( size <= std::numeric_limits::max(), tx_too_big, "packed_transaction is too big" ); return static_cast(size); } -uint32_t packed_transaction::get_prunable_size()const { +uint32_t packed_transaction_v0::get_prunable_size()const { uint64_t size = fc::raw::pack_size(signatures); size += packed_context_free_data.size(); EOS_ASSERT( size <= std::numeric_limits::max(), tx_too_big, "packed_transaction is too big" ); return static_cast(size); } -digest_type packed_transaction::packed_digest()const { +digest_type packed_transaction_v0::packed_digest()const { digest_type::encoder prunable; fc::raw::pack( prunable, signatures ); fc::raw::pack( prunable, packed_context_free_data ); @@ -256,21 +256,19 @@ static bytes zlib_compress_transaction(const transaction& t) { return out; } -bytes packed_transaction::get_raw_transaction() const +packed_transaction_v0::packed_transaction_v0(const bytes& packed_txn, const vector& sigs, const bytes& packed_cfd, compression_type _compression) +:signatures(sigs) +,compression(_compression) +,packed_context_free_data(packed_cfd) +,packed_trx(packed_txn) { - try { - switch(compression) { - case compression_type::none: - return packed_trx; - case compression_type::zlib: - return zlib_decompress(packed_trx); - default: - EOS_THROW(unknown_transaction_compression, "Unknown transaction compression algorithm"); - } - } FC_CAPTURE_AND_RETHROW((compression)(packed_trx)) + local_unpack_transaction({}); + if( !packed_context_free_data.empty() ) { + local_unpack_context_free_data(); + } } -packed_transaction::packed_transaction( bytes&& packed_txn, vector&& sigs, bytes&& packed_cfd, compression_type _compression ) +packed_transaction_v0::packed_transaction_v0( bytes&& packed_txn, vector&& sigs, bytes&& packed_cfd, compression_type _compression ) :signatures(std::move(sigs)) ,compression(_compression) ,packed_context_free_data(std::move(packed_cfd)) @@ -282,7 +280,7 @@ packed_transaction::packed_transaction( bytes&& packed_txn, vector&& sigs, vector&& cfd, compression_type _compression ) +packed_transaction_v0::packed_transaction_v0( bytes&& packed_txn, vector&& sigs, vector&& cfd, compression_type _compression ) :signatures(std::move(sigs)) ,compression(_compression) ,packed_trx(std::move(packed_txn)) @@ -293,7 +291,7 @@ packed_transaction::packed_transaction( bytes&& packed_txn, vector&& sigs, bytes&& packed_cfd, compression_type _compression ) +packed_transaction_v0::packed_transaction_v0( transaction&& t, vector&& sigs, bytes&& packed_cfd, compression_type _compression ) :signatures(std::move(sigs)) ,compression(_compression) ,packed_context_free_data(std::move(packed_cfd)) @@ -306,81 +304,343 @@ packed_transaction::packed_transaction( transaction&& t, vector& } } -void packed_transaction::reflector_init() +void packed_transaction_v0::reflector_init() { // called after construction, but always on the same thread and before packed_transaction passed to any other threads static_assert(fc::raw::has_feature_reflector_init_on_unpacked_reflected_types, "FC unpack needs to call reflector_init otherwise unpacked_trx will not be initialized"); - EOS_ASSERT( unpacked_trx.expiration == time_point_sec(), tx_decompression_error, "packed_transaction already unpacked" ); + EOS_ASSERT( unpacked_trx.expiration == time_point_sec(), tx_decompression_error, "packed_transaction_v0 already unpacked" ); local_unpack_transaction({}); local_unpack_context_free_data(); } -void packed_transaction::local_unpack_transaction(vector&& context_free_data) -{ +static transaction unpack_transaction(const bytes& packed_trx, packed_transaction_v0::compression_type compression) { try { switch( compression ) { - case compression_type::none: - unpacked_trx = signed_transaction( unpack_transaction( packed_trx ), signatures, std::move(context_free_data) ); - break; - case compression_type::zlib: - unpacked_trx = signed_transaction( zlib_decompress_transaction( packed_trx ), signatures, std::move(context_free_data) ); - break; + case packed_transaction_v0::compression_type::none: + return unpack_transaction( packed_trx ); + case packed_transaction_v0::compression_type::zlib: + return zlib_decompress_transaction( packed_trx ); default: EOS_THROW( unknown_transaction_compression, "Unknown transaction compression algorithm" ); } - trx_id = unpacked_trx.id(); } FC_CAPTURE_AND_RETHROW( (compression) ) } -void packed_transaction::local_unpack_context_free_data() +void packed_transaction_v0::local_unpack_transaction(vector&& context_free_data) +{ + unpacked_trx = signed_transaction( unpack_transaction( packed_trx, compression ), signatures, std::move(context_free_data) ); + trx_id = unpacked_trx.id(); +} + +static vector unpack_context_free_data(const bytes& packed_context_free_data, packed_transaction_v0::compression_type compression) { try { - EOS_ASSERT(unpacked_trx.context_free_data.empty(), tx_decompression_error, "packed_transaction.context_free_data not empty"); switch( compression ) { - case compression_type::none: - unpacked_trx.context_free_data = unpack_context_free_data( packed_context_free_data ); - break; - case compression_type::zlib: - unpacked_trx.context_free_data = zlib_decompress_context_free_data( packed_context_free_data ); - break; + case packed_transaction_v0::compression_type::none: + return unpack_context_free_data( packed_context_free_data ); + case packed_transaction_v0::compression_type::zlib: + return zlib_decompress_context_free_data( packed_context_free_data ); default: EOS_THROW( unknown_transaction_compression, "Unknown transaction compression algorithm" ); } } FC_CAPTURE_AND_RETHROW( (compression) ) } -void packed_transaction::local_pack_transaction() +void packed_transaction_v0::local_unpack_context_free_data() { + EOS_ASSERT(unpacked_trx.context_free_data.empty(), tx_decompression_error, "packed_transaction.context_free_data not empty"); + unpacked_trx.context_free_data = unpack_context_free_data( packed_context_free_data, compression ); +} + +static bytes pack_transaction(const transaction& trx, packed_transaction_v0::compression_type compression) { try { switch(compression) { - case compression_type::none: - packed_trx = pack_transaction(unpacked_trx); - break; - case compression_type::zlib: - packed_trx = zlib_compress_transaction(unpacked_trx); - break; + case packed_transaction_v0::compression_type::none: { + return pack_transaction( trx ); + } + case packed_transaction_v0::compression_type::zlib: + return zlib_compress_transaction(trx); default: EOS_THROW(unknown_transaction_compression, "Unknown transaction compression algorithm"); } } FC_CAPTURE_AND_RETHROW((compression)) } -void packed_transaction::local_pack_context_free_data() +void packed_transaction_v0::local_pack_transaction() { + packed_trx = pack_transaction(unpacked_trx, compression); +} + +static bytes pack_context_free_data( const vector& cfd, packed_transaction_v0::compression_type compression ) { try { switch(compression) { - case compression_type::none: - packed_context_free_data = pack_context_free_data(unpacked_trx.context_free_data); - break; - case compression_type::zlib: - packed_context_free_data = zlib_compress_context_free_data(unpacked_trx.context_free_data); - break; + case packed_transaction_v0::compression_type::none: + return pack_context_free_data(cfd); + case packed_transaction_v0::compression_type::zlib: + return zlib_compress_context_free_data(cfd); default: EOS_THROW(unknown_transaction_compression, "Unknown transaction compression algorithm"); } } FC_CAPTURE_AND_RETHROW((compression)) } +void packed_transaction_v0::local_pack_context_free_data() +{ + packed_context_free_data = pack_context_free_data(unpacked_trx.context_free_data, compression); +} + + +digest_type packed_transaction::prunable_data_type::none::prunable_digest() const { + return digest; +} + +digest_type packed_transaction::prunable_data_type::partial::prunable_digest() const { + EOS_THROW(tx_prune_exception, "unimplemented"); +} + +digest_type packed_transaction::prunable_data_type::full::prunable_digest() const { + EOS_THROW(tx_prune_exception, "unimplemented"); +} + +digest_type packed_transaction::prunable_data_type::full_legacy::prunable_digest() const { + digest_type::encoder prunable; + fc::raw::pack( prunable, this->signatures ); + fc::raw::pack( prunable, this->packed_context_free_data ); + return prunable.result(); +} + +digest_type packed_transaction::prunable_data_type::digest() const { + return std::visit( [](const auto& obj) { return obj.prunable_digest(); }, prunable_data ); +} + +packed_transaction::prunable_data_type packed_transaction::prunable_data_type::prune_all() const { + return std::visit([](const auto& obj) -> packed_transaction::prunable_data_type { return {none{obj.prunable_digest()}}; }, prunable_data); +} + +static constexpr std::size_t digest_pack_size = 32; + +static std::size_t padded_pack_size(const packed_transaction::prunable_data_type::none& obj, packed_transaction::prunable_data_type::compression_type) { + std::size_t result = fc::raw::pack_size(obj); + EOS_ASSERT(result == digest_pack_size, packed_transaction_type_exception, "Unexpected size of packed digest"); + return result; +} + + +static std::size_t padded_pack_size(const packed_transaction::prunable_data_type::partial& obj, packed_transaction::prunable_data_type::compression_type segment_compression) { + EOS_THROW(tx_prune_exception, "unimplemented"); +} + +static std::size_t padded_pack_size(const packed_transaction::prunable_data_type::full& obj, packed_transaction::prunable_data_type::compression_type segment_compression) { + EOS_THROW(tx_prune_exception, "unimplemented"); +#if 0 + std::size_t context_free_size = fc::raw::pack_size(fc::unsigned_int(obj.context_free_segments.size())); + context_free_size += obj.context_free_segments.size(); + for(const std::vector& v: obj.context_free_segments) { + // TODO: Handle segment_compression + context_free_size += std::max(fc::raw::pack_size(v), digest_pack_size); + } + return fc::raw::pack_size(obj.signatures) + std::max(context_free_size, digest_pack_size); +#endif +} + +static std::size_t padded_pack_size(const packed_transaction::prunable_data_type::full_legacy& obj, packed_transaction::prunable_data_type::compression_type) { + return std::max(fc::raw::pack_size(obj), digest_pack_size); +} + +std::size_t packed_transaction::prunable_data_type::maximum_pruned_pack_size(packed_transaction::prunable_data_type::compression_type compression) const { + return 1 + std::visit([&](const auto& t){ return padded_pack_size(t, compression); }, prunable_data); +} +static packed_transaction::prunable_data_type make_prunable_transaction_data( bool legacy, vector signatures, + vector context_free_data, + packed_transaction::compression_type _compression ) { + if(legacy) { + bytes packed_cfd = pack_context_free_data( context_free_data, _compression ); + return {packed_transaction::prunable_data_type::full_legacy{std::move( signatures), std::move( packed_cfd), std::move( context_free_data) } }; + } else { + return {packed_transaction::prunable_data_type::full{std::move( signatures), std::move( context_free_data) } }; + } +} + +packed_transaction::packed_transaction(signed_transaction t, bool legacy, compression_type _compression) + : compression(_compression), + prunable_data(make_prunable_transaction_data(legacy, std::move(t.signatures), std::move(t.context_free_data), _compression)), + packed_trx(pack_transaction(t, compression)), + unpacked_trx(std::move(t)), + trx_id(unpacked_trx.id()) +{ + estimated_size = calculate_estimated_size(); +} + +packed_transaction::packed_transaction(const packed_transaction_v0& other, bool legacy) + : compression(other.compression), + prunable_data(legacy ? prunable_data_type{prunable_data_type::full_legacy{other.signatures, + other.packed_context_free_data, + other.unpacked_trx.context_free_data } } + : prunable_data_type{prunable_data_type::full{other.signatures, + other.unpacked_trx.context_free_data } }), + packed_trx(other.packed_trx), + unpacked_trx(other.unpacked_trx), + trx_id(other.id()) +{ + EOS_ASSERT( legacy, transaction_exception, "Full type of prunable_data_type is not supported" ); + estimated_size = calculate_estimated_size(); +} + +packed_transaction::packed_transaction(packed_transaction_v0&& other, bool legacy) + : compression(other.compression), + prunable_data(legacy ? prunable_data_type{prunable_data_type::full_legacy{std::move(other.signatures), + std::move(other.packed_context_free_data), + std::move(other.unpacked_trx.context_free_data) } } + : prunable_data_type{prunable_data_type::full{std::move(other.signatures), + std::move(other.unpacked_trx.context_free_data) } }), + packed_trx(std::move(other.packed_trx)), + unpacked_trx(std::move(other.unpacked_trx)), + trx_id(other.id()) +{ + EOS_ASSERT( legacy, transaction_exception, "Full type of prunable_data_type is not supported" ); + estimated_size = calculate_estimated_size(); +} + +packed_transaction_v0_ptr packed_transaction::to_packed_transaction_v0() const { + const auto* sigs = get_signatures(); + EOS_ASSERT( std::holds_alternative(prunable_data.prunable_data), transaction_exception, "Failed to get full_legacy variant in to_packed_transaction_v0" ); + auto& legacy = std::get(prunable_data.prunable_data); + return std::make_shared( packed_trx, + sigs != nullptr ? *sigs : vector(), + legacy.packed_context_free_data, + compression); +} + +uint32_t packed_transaction::get_unprunable_size()const { + uint64_t size = config::fixed_net_overhead_of_packed_trx; + size += packed_trx.size(); + EOS_ASSERT( size <= std::numeric_limits::max(), tx_too_big, "packed_transaction is too big" ); + return static_cast(size); +} + +static uint32_t get_prunable_size_impl(const packed_transaction::prunable_data_type::full_legacy& obj) { + uint64_t size = fc::raw::pack_size(obj.signatures); + size += obj.packed_context_free_data.size(); + EOS_ASSERT( size <= std::numeric_limits::max(), tx_too_big, "packed_transaction is too big" ); + return static_cast(size); +} + +static uint32_t get_prunable_size_impl(const packed_transaction::prunable_data_type::full&) { + EOS_THROW(tx_prune_exception, "unimplemented"); +} + +template +static uint32_t get_prunable_size_impl(const T&) { + EOS_THROW(tx_prune_exception, "unknown size: prunable data has been pruned."); +} + +uint32_t packed_transaction::get_prunable_size()const { + return std::visit([](const auto& obj) { return get_prunable_size_impl(obj); }, prunable_data.prunable_data); +} + +uint32_t packed_transaction::calculate_estimated_size() const { + auto est_size = overloaded{ + [](const std::vector& vec) { + uint32_t s = 0; + for( const auto& v : vec ) s += v.size(); + return s; + }, + [](const std::vector& vec) { + uint32_t s = 0; + for( const auto& v : vec ) { + s += std::visit( overloaded{ + [](const digest_type& t) { return sizeof(t); }, + [](const bytes& vec) { return vec.size(); } + }, v ); + } + return s; + } + }; + auto visitor = overloaded{ + [](const prunable_data_type::none& v) { return 0ul; }, + [&](const prunable_data_type::partial& v) { + return v.signatures.size() * sizeof(signature_type) + est_size(v.context_free_segments); + }, + [&](const prunable_data_type::full& v) { + return v.signatures.size() * sizeof(signature_type) + est_size(v.context_free_segments); + }, + [&](const prunable_data_type::full_legacy& v) { + return v.signatures.size() * sizeof(signature_type) + v.packed_context_free_data.size() + est_size(v.context_free_segments); + } + }; + + return sizeof(*this) + packed_trx.size() * 2 + std::visit(visitor, prunable_data.prunable_data); +} + +digest_type packed_transaction::packed_digest()const { + digest_type::encoder enc; + fc::raw::pack( enc, compression ); + fc::raw::pack( enc, packed_trx ); + fc::raw::pack( enc, prunable_data.digest() ); + return enc.result(); +} + +template +static auto maybe_get_signatures(const T& obj) -> decltype(&obj.signatures) { return &obj.signatures; } +static auto maybe_get_signatures(const packed_transaction::prunable_data_type::none&) -> const std::vector* { return nullptr; } +const vector* packed_transaction::get_signatures()const { + return std::visit([](const auto& obj) { return maybe_get_signatures(obj); }, prunable_data.prunable_data); +} + +const vector* packed_transaction::get_context_free_data()const { + if( std::holds_alternative(prunable_data.prunable_data) ) { + return &std::get(prunable_data.prunable_data).context_free_segments; + } else if( std::holds_alternative(prunable_data.prunable_data) ) { + return &std::get(prunable_data.prunable_data).context_free_segments; + } else { + return nullptr; + } +} + +const bytes* maybe_get_context_free_data(const packed_transaction::prunable_data_type::none&, std::size_t) { return nullptr; } +const bytes* maybe_get_context_free_data(const packed_transaction::prunable_data_type::partial& p, std::size_t i) { + if( p.context_free_segments.size() <= i ) return nullptr; + return std::visit( + overloaded{ + []( const digest_type& t ) -> const bytes* { return nullptr; }, + []( const bytes& vec ) { return &vec; } + }, p.context_free_segments[i] ); +} +const bytes* maybe_get_context_free_data(const packed_transaction::prunable_data_type::full_legacy& full_leg, std::size_t i) { + if( full_leg.context_free_segments.size() <= i ) return nullptr; + return &full_leg.context_free_segments[i]; +} +const bytes* maybe_get_context_free_data(const packed_transaction::prunable_data_type::full& f, std::size_t i) { + if( f.context_free_segments.size() <= i ) return nullptr; + return &f.context_free_segments[i]; +} + +const bytes* packed_transaction::get_context_free_data(std::size_t segment_ordinal) const { + return std::visit([&](const auto& obj) { return maybe_get_context_free_data(obj, segment_ordinal); }, prunable_data.prunable_data); +} + +void packed_transaction::prune_all() { + prunable_data = prunable_data.prune_all(); +} + +std::size_t packed_transaction::maximum_pruned_pack_size(cf_compression_type segment_compression) const { + return fc::raw::pack_size(compression) + fc::raw::pack_size(packed_trx) + prunable_data.maximum_pruned_pack_size(segment_compression); +} + +void packed_transaction::reflector_init() +{ + // called after construction, but always on the same thread and before packed_transaction passed to any other threads + static_assert(fc::raw::has_feature_reflector_init_on_unpacked_reflected_types, + "FC unpack needs to call reflector_init otherwise unpacked_trx will not be initialized"); + EOS_ASSERT( unpacked_trx.expiration == time_point_sec(), tx_decompression_error, "packed_transaction already unpacked" ); + unpacked_trx = unpack_transaction(packed_trx, compression); + trx_id = unpacked_trx.id(); + if( std::holds_alternative(prunable_data.prunable_data) ) { + auto& legacy = std::get(prunable_data.prunable_data); + legacy.context_free_segments = unpack_context_free_data( legacy.packed_context_free_data, compression ); + } + estimated_size = calculate_estimated_size(); +} } } // eosio::chain diff --git a/libraries/chain/transaction_context.cpp b/libraries/chain/transaction_context.cpp index 63a6a78ea6e..0d6e1a1ab0d 100644 --- a/libraries/chain/transaction_context.cpp +++ b/libraries/chain/transaction_context.cpp @@ -44,28 +44,22 @@ namespace eosio { namespace chain { } transaction_context::transaction_context( controller& c, - const signed_transaction& t, - const transaction_id_type& trx_id, + const packed_transaction& t, transaction_checktime_timer&& tmr, fc::time_point s ) :control(c) - ,trx(t) - ,id(trx_id) - ,undo_session() + ,packed_trx(t) + ,undo_session(!c.skip_db_sessions() ? c.kv_db().make_session() : c.kv_db().make_no_op_session()) ,trace(std::make_shared()) ,start(s) ,transaction_timer(std::move(tmr)) ,net_usage(trace->net_usage) ,pseudo_start(s) { - if (!c.skip_db_sessions()) { - undo_session = c.mutable_db().start_undo_session(true); - } - trace->id = id; + trace->id = packed_trx.id(); trace->block_num = c.head_block_num() + 1; trace->block_time = c.pending_block_time(); trace->producer_block_id = c.pending_producer_block_id(); - executed.reserve( trx.total_actions() ); } void transaction_context::disallow_transaction_extensions( const char* error_msg )const { @@ -76,10 +70,9 @@ namespace eosio { namespace chain { } } - void transaction_context::init(uint64_t initial_net_usage) + void transaction_context::init( uint64_t initial_net_usage ) { EOS_ASSERT( !is_initialized, transaction_exception, "cannot initialize twice" ); - const static int64_t large_number_no_overflow = std::numeric_limits::max()/2; const auto& cfg = control.get_global_properties().configuration; auto& rl = control.get_mutable_resource_limits_manager(); @@ -102,6 +95,7 @@ namespace eosio { namespace chain { _deadline = start + objective_duration_limit; } + const transaction& trx = packed_trx.get_transaction(); // Possibly lower net_limit to optional limit set in the transaction header uint64_t trx_specified_net_usage_limit = static_cast(trx.max_net_usage_words.value) * 8; if( trx_specified_net_usage_limit > 0 && trx_specified_net_usage_limit <= net_limit ) { @@ -149,7 +143,7 @@ namespace eosio { namespace chain { eager_net_limit = net_limit; - // Possible lower eager_net_limit to what the billed accounts can pay plus some (objective) leeway + // Possibly lower eager_net_limit to what the billed accounts can pay plus some (objective) leeway auto new_eager_net_limit = std::min( eager_net_limit, static_cast(account_net_limit + cfg.net_usage_leeway) ); if( new_eager_net_limit < eager_net_limit ) { eager_net_limit = new_eager_net_limit; @@ -198,18 +192,20 @@ namespace eosio { namespace chain { void transaction_context::init_for_implicit_trx( uint64_t initial_net_usage ) { + const transaction& trx = packed_trx.get_transaction(); if( trx.transaction_extensions.size() > 0 ) { disallow_transaction_extensions( "no transaction extensions supported yet for implicit transactions" ); } published = control.pending_block_time(); - init( initial_net_usage); + init( initial_net_usage ); } void transaction_context::init_for_input_trx( uint64_t packed_trx_unprunable_size, uint64_t packed_trx_prunable_size, bool skip_recording ) { + const transaction& trx = packed_trx.get_transaction(); if( trx.transaction_extensions.size() > 0 ) { disallow_transaction_extensions( "no transaction extensions supported yet for input transactions" ); } @@ -236,20 +232,41 @@ namespace eosio { namespace chain { + static_cast(config::transaction_id_net_usage); } + init_for_input_trx_common( initial_net_usage, skip_recording ); + } + + void transaction_context::init_for_input_trx_with_explicit_net( uint32_t explicit_net_usage_words, + bool skip_recording ) + { + const transaction& trx = packed_trx.get_transaction(); + if( trx.transaction_extensions.size() > 0 ) { + disallow_transaction_extensions( "no transaction extensions supported yet for input transactions" ); + } + + explicit_net_usage = true; + net_usage = (static_cast(explicit_net_usage_words) * 8); + + init_for_input_trx_common( 0, skip_recording ); + } + + void transaction_context::init_for_input_trx_common( uint64_t initial_net_usage, bool skip_recording ) + { published = control.pending_block_time(); is_input = true; + const transaction& trx = packed_trx.get_transaction(); if (!control.skip_trx_checks()) { control.validate_expiration(trx); control.validate_tapos(trx); validate_referenced_accounts( trx, enforce_whiteblacklist && control.is_producing_block() ); } - init( initial_net_usage); + init( initial_net_usage ); if (!skip_recording) - record_transaction( id, trx.expiration ); /// checks for dupes + record_transaction( packed_trx.id(), trx.expiration ); /// checks for dupes } void transaction_context::init_for_deferred_trx( fc::time_point p ) { + const transaction& trx = packed_trx.get_transaction(); if( (trx.expiration.sec_since_epoch() != 0) && (trx.transaction_extensions.size() > 0) ) { disallow_transaction_extensions( "no transaction extensions supported yet for deferred transactions" ); } @@ -267,6 +284,7 @@ namespace eosio { namespace chain { void transaction_context::exec() { EOS_ASSERT( is_initialized, transaction_exception, "must first initialize" ); + const transaction& trx = packed_trx.get_transaction(); if( apply_context_free ) { for( const auto& act : trx.context_free_actions ) { schedule_action( act, act.account, true, 0, 0 ); @@ -294,6 +312,7 @@ namespace eosio { namespace chain { EOS_ASSERT( is_initialized, transaction_exception, "must first initialize" ); if( is_input ) { + const transaction& trx = packed_trx.get_transaction(); auto& am = control.get_mutable_authorization_manager(); for( const auto& act : trx.actions ) { for( const auto& auth : act.authorization ) { @@ -329,10 +348,10 @@ namespace eosio { namespace chain { billing_timer_exception_code = tx_cpu_usage_exceeded::code_value; } - net_usage = ((net_usage + 7)/8)*8; // Round up to nearest multiple of word size (8 bytes) - eager_net_limit = net_limit; - check_net_usage(); + + round_up_net_usage(); // Round up to nearest multiple of word size (8 bytes). + check_net_usage(); // Check that NET usage satisfies limits (even when explicit_net_usage is true). auto now = fc::time_point::now(); trace->elapsed = now - start; @@ -346,29 +365,27 @@ namespace eosio { namespace chain { } void transaction_context::squash() { - if (undo_session) undo_session->squash(); + undo_session.squash(); } void transaction_context::undo() { - if (undo_session) undo_session->undo(); + undo_session.undo(); } void transaction_context::check_net_usage()const { - if (!control.skip_trx_checks()) { - if( BOOST_UNLIKELY(net_usage > eager_net_limit) ) { - if ( net_limit_due_to_block ) { - EOS_THROW( block_net_usage_exceeded, - "not enough space left in block: ${net_usage} > ${net_limit}", - ("net_usage", net_usage)("net_limit", eager_net_limit) ); - } else if (net_limit_due_to_greylist) { - EOS_THROW( greylist_net_usage_exceeded, - "greylisted transaction net usage is too high: ${net_usage} > ${net_limit}", - ("net_usage", net_usage)("net_limit", eager_net_limit) ); - } else { - EOS_THROW( tx_net_usage_exceeded, - "transaction net usage is too high: ${net_usage} > ${net_limit}", - ("net_usage", net_usage)("net_limit", eager_net_limit) ); - } + if( BOOST_UNLIKELY(net_usage > eager_net_limit) ) { + if ( net_limit_due_to_block ) { + EOS_THROW( block_net_usage_exceeded, + "not enough space left in block: ${net_usage} > ${net_limit}", + ("net_usage", net_usage)("net_limit", eager_net_limit) ); + } else if (net_limit_due_to_greylist) { + EOS_THROW( greylist_net_usage_exceeded, + "greylisted transaction net usage is too high: ${net_usage} > ${net_limit}", + ("net_usage", net_usage)("net_limit", eager_net_limit) ); + } else { + EOS_THROW( tx_net_usage_exceeded, + "transaction net usage is too high: ${net_usage} > ${net_limit}", + ("net_usage", net_usage)("net_limit", eager_net_limit) ); } } } @@ -504,9 +521,9 @@ namespace eosio { namespace chain { } } - void transaction_context::add_ram_usage( account_name account, int64_t ram_delta ) { + void transaction_context::add_ram_usage( account_name account, int64_t ram_delta, const storage_usage_trace& trace ) { auto& rl = control.get_mutable_resource_limits_manager(); - rl.add_pending_ram_usage( account, ram_delta ); + rl.add_pending_ram_usage( account, ram_delta, trace ); if( ram_delta > 0 ) { validate_ram_usage.insert( account ); } @@ -625,6 +642,15 @@ namespace eosio { namespace chain { void transaction_context::execute_action( uint32_t action_ordinal, uint32_t recurse_depth ) { apply_context acontext( control, *this, action_ordinal, recurse_depth ); + + if (recurse_depth == 0) { + if (auto dm_logger = control.get_deep_mind_logger()) { + fc_dlog(*dm_logger, "CREATION_OP ROOT ${action_id}", + ("action_id", get_action_id()) + ); + } + } + acontext.exec(); } @@ -632,6 +658,7 @@ namespace eosio { namespace chain { void transaction_context::schedule_transaction() { // Charge ahead of time for the additional net usage needed to retire the delayed transaction // whether that be by successfully executing, soft failure, hard failure, or expiration. + const transaction& trx = packed_trx.get_transaction(); if( trx.delay_sec.value == 0 ) { // Do not double bill. Only charge if we have not already charged for the delay. const auto& cfg = control.get_global_properties().configuration; add_net_usage( static_cast(cfg.base_per_transaction_net_usage) @@ -640,9 +667,10 @@ namespace eosio { namespace chain { auto first_auth = trx.first_authorizer(); + std::string event_id; uint32_t trx_size = 0; const auto& cgto = control.mutable_db().create( [&]( auto& gto ) { - gto.trx_id = id; + gto.trx_id = packed_trx.id(); gto.payer = first_auth; gto.sender = account_name(); /// delayed transactions have no sender gto.sender_id = transaction_id_to_sender_id( gto.trx_id ); @@ -650,10 +678,27 @@ namespace eosio { namespace chain { gto.delay_until = gto.published + delay; gto.expiration = gto.delay_until + fc::seconds(control.get_global_properties().configuration.deferred_trx_expiration_window); trx_size = gto.set( trx ); + + if (auto dm_logger = control.get_deep_mind_logger()) { + event_id = STORAGE_EVENT_ID("${id}", ("id", gto.id)); + + auto packed_signed_trx = fc::raw::pack(packed_trx.to_packed_transaction_v0()->get_signed_transaction()); + fc_dlog(*dm_logger, "DTRX_OP PUSH_CREATE ${action_id} ${sender} ${sender_id} ${payer} ${published} ${delay} ${expiration} ${trx_id} ${trx}", + ("action_id", get_action_id()) + ("sender", gto.sender) + ("sender_id", gto.sender_id) + ("payer", gto.payer) + ("published", gto.published) + ("delay", gto.delay_until) + ("expiration", gto.expiration) + ("trx_id", gto.trx_id) + ("trx", fc::to_hex(packed_signed_trx.data(), packed_signed_trx.size())) + ); + } }); int64_t ram_delta = (config::billable_size_v + trx_size); - add_ram_usage( cgto.payer, ram_delta ); + add_ram_usage( cgto.payer, ram_delta, storage_usage_trace(get_action_id(), std::move(event_id), "deferred_trx", "push", "deferred_trx_pushed") ); trace->account_ram_delta = account_delta( cgto.payer, ram_delta ); } @@ -675,12 +720,14 @@ namespace eosio { namespace chain { const auto& db = control.db(); const auto& auth_manager = control.get_authorization_manager(); - for( const auto& a : trx.context_free_actions ) { - auto* code = db.find(a.account); - EOS_ASSERT( code != nullptr, transaction_exception, - "action's code account '${account}' does not exist", ("account", a.account) ); - EOS_ASSERT( a.authorization.size() == 0, transaction_exception, - "context-free actions cannot have authorizations" ); + if( !trx.context_free_actions.empty() && !control.skip_trx_checks() ) { + for( const auto& a : trx.context_free_actions ) { + auto* code = db.find( a.account ); + EOS_ASSERT( code != nullptr, transaction_exception, + "action's code account '${account}' does not exist", ("account", a.account) ); + EOS_ASSERT( a.authorization.size() == 0, transaction_exception, + "context-free actions cannot have authorizations" ); + } } flat_set actors; diff --git a/libraries/chain/transaction_metadata.cpp b/libraries/chain/transaction_metadata.cpp index 00a9e2c329a..d6d1861e50d 100644 --- a/libraries/chain/transaction_metadata.cpp +++ b/libraries/chain/transaction_metadata.cpp @@ -13,13 +13,20 @@ recover_keys_future transaction_metadata::start_recover_keys( packed_transaction return async_thread_pool( thread_pool, [trx{std::move(trx)}, chain_id, time_limit, max_variable_sig_size]() mutable { fc::time_point deadline = time_limit == fc::microseconds::maximum() ? fc::time_point::maximum() : fc::time_point::now() + time_limit; - check_variable_sig_size( trx, max_variable_sig_size ); - const signed_transaction& trn = trx->get_signed_transaction(); + const vector& sigs = check_variable_sig_size( trx, max_variable_sig_size ); + const vector* context_free_data = trx->get_context_free_data(); + EOS_ASSERT( context_free_data, tx_no_context_free_data, "context free data pruned from packed_transaction" ); flat_set recovered_pub_keys; - fc::microseconds cpu_usage = trn.get_signature_keys( chain_id, deadline, recovered_pub_keys ); + const bool allow_duplicate_keys = false; + fc::microseconds cpu_usage = + trx->get_transaction().get_signature_keys(sigs, chain_id, deadline, *context_free_data, recovered_pub_keys, allow_duplicate_keys); return std::make_shared( private_type(), std::move( trx ), cpu_usage, std::move( recovered_pub_keys ) ); } ); } +uint32_t transaction_metadata::get_estimated_size() const { + return sizeof(*this) + _recovered_pub_keys.size() * sizeof(public_key_type) + packed_trx()->get_estimated_size(); +} + } } // eosio::chain diff --git a/libraries/chain/wasm_config.cpp b/libraries/chain/wasm_config.cpp new file mode 100644 index 00000000000..2968b0f1337 --- /dev/null +++ b/libraries/chain/wasm_config.cpp @@ -0,0 +1,15 @@ +#include +#include + +using namespace eosio::chain; + +void wasm_config::validate() const { + EOS_ASSERT(max_section_elements >= 4, action_validate_exception, "max_section_elements cannot be less than 4"); + EOS_ASSERT(max_func_local_bytes >= 8, action_validate_exception, "max_func_local_bytes cannot be less than 8"); + EOS_ASSERT(max_nested_structures >= 1, action_validate_exception, "max_nested_structures cannot be less than 1"); + EOS_ASSERT(max_symbol_bytes >= 32, action_validate_exception, "max_symbol_bytes cannot be less than 32"); + EOS_ASSERT(max_module_bytes >= 256, action_validate_exception, "max_module_bytes cannot be less than 256"); + EOS_ASSERT(max_code_bytes >= 32, action_validate_exception, "max_code_bytes cannot be less than 32"); + EOS_ASSERT(max_pages >= 1, action_validate_exception, "max_pages cannot be less than 1"); + EOS_ASSERT(max_call_depth >= 2, action_validate_exception, "max_call_depth cannot be less than 2"); +} diff --git a/libraries/chain/wasm_eosio_injection.cpp b/libraries/chain/wasm_eosio_injection.cpp index 2c627e13ea7..73eab67ba1d 100644 --- a/libraries/chain/wasm_eosio_injection.cpp +++ b/libraries/chain/wasm_eosio_injection.cpp @@ -29,11 +29,6 @@ void data_segments_injection_visitor::inject( Module& m ) { } void data_segments_injection_visitor::initializer() { } -void max_memory_injection_visitor::inject( Module& m ) { - if(m.memories.defs.size() && m.memories.defs[0].type.size.max > maximum_linear_memory/wasm_page_size) - m.memories.defs[0].type.size.max = maximum_linear_memory/wasm_page_size; -} -void max_memory_injection_visitor::initializer() {} int32_t call_depth_check_and_insert_checktime::global_idx = -1; uint32_t instruction_counter::icnt = 0; diff --git a/libraries/chain/wasm_interface.cpp b/libraries/chain/wasm_interface.cpp index c29bb77838c..5a53207bc3a 100644 --- a/libraries/chain/wasm_interface.cpp +++ b/libraries/chain/wasm_interface.cpp @@ -1,3 +1,5 @@ +#include +#include #include #include #include @@ -30,7 +32,6 @@ #endif namespace eosio { namespace chain { - using namespace webassembly::common; wasm_interface::wasm_interface(vm_type vm, bool eosvmoc_tierup, const chainbase::database& d, const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config) : my( new wasm_interface_impl(vm, eosvmoc_tierup, d, data_dir, eosvmoc_config) ) {} @@ -38,6 +39,13 @@ namespace eosio { namespace chain { wasm_interface::~wasm_interface() {} void wasm_interface::validate(const controller& control, const bytes& code) { + const auto& pso = control.db().get(); + + if (control.is_builtin_activated(builtin_protocol_feature_t::configurable_wasm_limits)) { + const auto& gpo = control.get_global_properties(); + webassembly::eos_vm_runtime::validate( code, gpo.wasm_configuration, pso.whitelisted_intrinsics ); + return; + } Module module; try { Serialization::MemoryInputStream stream((U8*)code.data(), code.size()); @@ -51,10 +59,7 @@ namespace eosio { namespace chain { wasm_validations::wasm_binary_validation validator(control, module); validator.validate(); - const auto& pso = control.db().get(); - - root_resolver resolver( pso.whitelisted_intrinsics ); - LinkResult link_result = linkModule(module, resolver); + webassembly::eos_vm_runtime::validate( code, pso.whitelisted_intrinsics ); //there are a couple opportunties for improvement here-- //Easy: Cache the Module created here so it can be reused for instantiaion @@ -104,1984 +109,10 @@ namespace eosio { namespace chain { wasm_instantiated_module_interface::~wasm_instantiated_module_interface() {} wasm_runtime_interface::~wasm_runtime_interface() {} -#if defined(assert) - #undef assert -#endif - -class context_aware_api { - public: - context_aware_api(apply_context& ctx, bool context_free = false ) - :context(ctx) - { - if( context.is_context_free() ) - EOS_ASSERT( context_free, unaccessible_api, "only context free api's can be used in this context" ); - } - - void checktime() { - context.trx_context.checktime(); - } - - protected: - apply_context& context; - -}; - -class context_free_api : public context_aware_api { - public: - context_free_api( apply_context& ctx ) - :context_aware_api(ctx, true) { - /* the context_free_data is not available during normal application because it is prunable */ - EOS_ASSERT( context.is_context_free(), unaccessible_api, "this API may only be called from context_free apply" ); - } - - int get_context_free_data( uint32_t index, array_ptr buffer, uint32_t buffer_size )const { - return context.get_context_free_data( index, buffer, buffer_size ); - } -}; - -class privileged_api : public context_aware_api { - public: - privileged_api( apply_context& ctx ) - :context_aware_api(ctx) - { - EOS_ASSERT( context.is_privileged(), unaccessible_api, "${code} does not have permission to call this API", ("code",context.get_receiver()) ); - } - - /** - * This should return true if a feature is active and irreversible, false if not. - * - * Irreversiblity by fork-database is not consensus safe, therefore, this defines - * irreversiblity only by block headers not by BFT short-cut. - */ - int is_feature_active( int64_t feature_name ) { - return false; - } - - /** - * This should schedule the feature to be activated once the - * block that includes this call is irreversible. It should - * fail if the feature is already pending. - * - * Feature name should be base32 encoded name. - */ - void activate_feature( int64_t feature_name ) { - EOS_ASSERT( false, unsupported_feature, "Unsupported Hardfork Detected" ); - } - - /** - * Pre-activates the specified protocol feature. - * Fails if the feature is unrecognized, disabled, or not allowed to be activated at the current time. - * Also fails if the feature was already activated or pre-activated. - */ - void preactivate_feature( const digest_type& feature_digest ) { - context.control.preactivate_feature( feature_digest ); - } - - /** - * update the resource limits associated with an account. Note these new values will not take effect until the - * next resource "tick" which is currently defined as a cycle boundary inside a block. - * - * @param account - the account whose limits are being modified - * @param ram_bytes - the limit for ram bytes - * @param net_weight - the weight for determining share of network capacity - * @param cpu_weight - the weight for determining share of compute capacity - */ - void set_resource_limits( account_name account, int64_t ram_bytes, int64_t net_weight, int64_t cpu_weight) { - EOS_ASSERT(ram_bytes >= -1, wasm_execution_error, "invalid value for ram resource limit expected [-1,INT64_MAX]"); - EOS_ASSERT(net_weight >= -1, wasm_execution_error, "invalid value for net resource weight expected [-1,INT64_MAX]"); - EOS_ASSERT(cpu_weight >= -1, wasm_execution_error, "invalid value for cpu resource weight expected [-1,INT64_MAX]"); - if( context.control.get_mutable_resource_limits_manager().set_account_limits(account, ram_bytes, net_weight, cpu_weight) ) { - context.trx_context.validate_ram_usage.insert( account ); - } - } - - void get_resource_limits( account_name account, int64_t& ram_bytes, int64_t& net_weight, int64_t& cpu_weight ) { - context.control.get_resource_limits_manager().get_account_limits( account, ram_bytes, net_weight, cpu_weight); - } - - int64_t set_proposed_producers_common( vector && producers, bool validate_keys ) { - EOS_ASSERT(producers.size() <= config::max_producers, wasm_execution_error, "Producer schedule exceeds the maximum producer count for this chain"); - EOS_ASSERT( producers.size() > 0 - || !context.control.is_builtin_activated( builtin_protocol_feature_t::disallow_empty_producer_schedule ), - wasm_execution_error, - "Producer schedule cannot be empty" - ); - - const auto num_supported_key_types = context.db.get().num_supported_key_types; - - // check that producers are unique - std::set unique_producers; - for (const auto& p: producers) { - EOS_ASSERT( context.is_account(p.producer_name), wasm_execution_error, "producer schedule includes a nonexisting account" ); - - p.authority.visit([&p, num_supported_key_types, validate_keys](const auto& a) { - uint32_t sum_weights = 0; - std::set unique_keys; - for (const auto& kw: a.keys ) { - EOS_ASSERT( kw.key.which() < num_supported_key_types, unactivated_key_type, - "Unactivated key type used in proposed producer schedule"); - - if( validate_keys ) { - EOS_ASSERT( kw.key.valid(), wasm_execution_error, "producer schedule includes an invalid key" ); - } - - if (std::numeric_limits::max() - sum_weights <= kw.weight) { - sum_weights = std::numeric_limits::max(); - } else { - sum_weights += kw.weight; - } - - unique_keys.insert(kw.key); - } - - EOS_ASSERT( a.keys.size() == unique_keys.size(), wasm_execution_error, "producer schedule includes a duplicated key for ${account}", ("account", p.producer_name)); - EOS_ASSERT( a.threshold > 0, wasm_execution_error, "producer schedule includes an authority with a threshold of 0 for ${account}", ("account", p.producer_name)); - EOS_ASSERT( sum_weights >= a.threshold, wasm_execution_error, "producer schedule includes an unsatisfiable authority for ${account}", ("account", p.producer_name)); - }); - - - unique_producers.insert(p.producer_name); - } - EOS_ASSERT( producers.size() == unique_producers.size(), wasm_execution_error, "duplicate producer name in producer schedule" ); - - return context.control.set_proposed_producers( std::move(producers) ); - } - - int64_t set_proposed_producers( array_ptr packed_producer_schedule, uint32_t datalen ) { - datastream ds( packed_producer_schedule, datalen ); - vector producers; - - vector old_version; - fc::raw::unpack(ds, old_version); - - /* - * Up-convert the producers - */ - for ( const auto& p: old_version ) { - producers.emplace_back(producer_authority{ p.producer_name, block_signing_authority_v0{ 1, {{p.block_signing_key, 1}} } } ); - } - - return set_proposed_producers_common(std::move(producers), true); - } - - int64_t set_proposed_producers_ex( uint64_t packed_producer_format, array_ptr packed_producer_schedule, uint32_t datalen ) { - if (packed_producer_format == 0) { - return set_proposed_producers(packed_producer_schedule, datalen); - } else if (packed_producer_format == 1) { - datastream ds( packed_producer_schedule, datalen ); - vector producers; - - fc::raw::unpack(ds, producers); - return set_proposed_producers_common(std::move(producers), false); - } else { - EOS_THROW(wasm_execution_error, "Producer schedule is in an unknown format!"); - } - } - - uint32_t get_blockchain_parameters_packed( array_ptr packed_blockchain_parameters, uint32_t buffer_size) { - auto& gpo = context.control.get_global_properties(); - - auto s = fc::raw::pack_size( gpo.configuration ); - if( buffer_size == 0 ) return s; - - if ( s <= buffer_size ) { - datastream ds( packed_blockchain_parameters, s ); - fc::raw::pack(ds, gpo.configuration); - return s; - } - return 0; - } - - void set_blockchain_parameters_packed( array_ptr packed_blockchain_parameters, uint32_t datalen) { - datastream ds( packed_blockchain_parameters, datalen ); - chain::chain_config cfg; - fc::raw::unpack(ds, cfg); - cfg.validate(); - context.db.modify( context.control.get_global_properties(), - [&]( auto& gprops ) { - gprops.configuration = cfg; - }); - } - - bool is_privileged( account_name n )const { - return context.db.get( n ).is_privileged(); - } - - void set_privileged( account_name n, bool is_priv ) { - const auto& a = context.db.get( n ); - context.db.modify( a, [&]( auto& ma ){ - ma.set_privileged( is_priv ); - }); - } - -}; - -class softfloat_api : public context_aware_api { - public: - // TODO add traps on truncations for special cases (NaN or outside the range which rounds to an integer) - softfloat_api( apply_context& ctx ) - :context_aware_api(ctx, true) {} - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wstrict-aliasing" - // float binops - float _eosio_f32_add( float a, float b ) { - float32_t ret = ::f32_add( to_softfloat32(a), to_softfloat32(b) ); - return *reinterpret_cast(&ret); - } - float _eosio_f32_sub( float a, float b ) { - float32_t ret = ::f32_sub( to_softfloat32(a), to_softfloat32(b) ); - return *reinterpret_cast(&ret); - } - float _eosio_f32_div( float a, float b ) { - float32_t ret = ::f32_div( to_softfloat32(a), to_softfloat32(b) ); - return *reinterpret_cast(&ret); - } - float _eosio_f32_mul( float a, float b ) { - float32_t ret = ::f32_mul( to_softfloat32(a), to_softfloat32(b) ); - return *reinterpret_cast(&ret); - } -#pragma GCC diagnostic pop - float _eosio_f32_min( float af, float bf ) { - float32_t a = to_softfloat32(af); - float32_t b = to_softfloat32(bf); - if (is_nan(a)) { - return af; - } - if (is_nan(b)) { - return bf; - } - if ( f32_sign_bit(a) != f32_sign_bit(b) ) { - return f32_sign_bit(a) ? af : bf; - } - return ::f32_lt(a,b) ? af : bf; - } - float _eosio_f32_max( float af, float bf ) { - float32_t a = to_softfloat32(af); - float32_t b = to_softfloat32(bf); - if (is_nan(a)) { - return af; - } - if (is_nan(b)) { - return bf; - } - if ( f32_sign_bit(a) != f32_sign_bit(b) ) { - return f32_sign_bit(a) ? bf : af; - } - return ::f32_lt( a, b ) ? bf : af; - } - float _eosio_f32_copysign( float af, float bf ) { - float32_t a = to_softfloat32(af); - float32_t b = to_softfloat32(bf); - uint32_t sign_of_b = b.v >> 31; - a.v &= ~(1 << 31); // clear the sign bit - a.v = a.v | (sign_of_b << 31); // add the sign of b - return from_softfloat32(a); - } - // float unops - float _eosio_f32_abs( float af ) { - float32_t a = to_softfloat32(af); - a.v &= ~(1 << 31); - return from_softfloat32(a); - } - float _eosio_f32_neg( float af ) { - float32_t a = to_softfloat32(af); - uint32_t sign = a.v >> 31; - a.v &= ~(1 << 31); - a.v |= (!sign << 31); - return from_softfloat32(a); - } - float _eosio_f32_sqrt( float a ) { - float32_t ret = ::f32_sqrt( to_softfloat32(a) ); - return from_softfloat32(ret); - } - // ceil, floor, trunc and nearest are lifted from libc - float _eosio_f32_ceil( float af ) { - float32_t a = to_softfloat32(af); - int e = (int)(a.v >> 23 & 0xFF) - 0X7F; - uint32_t m; - if (e >= 23) - return af; - if (e >= 0) { - m = 0x007FFFFF >> e; - if ((a.v & m) == 0) - return af; - if (a.v >> 31 == 0) - a.v += m; - a.v &= ~m; - } else { - if (a.v >> 31) - a.v = 0x80000000; // return -0.0f - else if (a.v << 1) - a.v = 0x3F800000; // return 1.0f - } - - return from_softfloat32(a); - } - float _eosio_f32_floor( float af ) { - float32_t a = to_softfloat32(af); - int e = (int)(a.v >> 23 & 0xFF) - 0X7F; - uint32_t m; - if (e >= 23) - return af; - if (e >= 0) { - m = 0x007FFFFF >> e; - if ((a.v & m) == 0) - return af; - if (a.v >> 31) - a.v += m; - a.v &= ~m; - } else { - if (a.v >> 31 == 0) - a.v = 0; - else if (a.v << 1) - a.v = 0xBF800000; // return -1.0f - } - return from_softfloat32(a); - } - float _eosio_f32_trunc( float af ) { - float32_t a = to_softfloat32(af); - int e = (int)(a.v >> 23 & 0xff) - 0x7f + 9; - uint32_t m; - if (e >= 23 + 9) - return af; - if (e < 9) - e = 1; - m = -1U >> e; - if ((a.v & m) == 0) - return af; - a.v &= ~m; - return from_softfloat32(a); - } - float _eosio_f32_nearest( float af ) { - float32_t a = to_softfloat32(af); - int e = a.v>>23 & 0xff; - int s = a.v>>31; - float32_t y; - if (e >= 0x7f+23) - return af; - if (s) - y = ::f32_add( ::f32_sub( a, float32_t{inv_float_eps} ), float32_t{inv_float_eps} ); - else - y = ::f32_sub( ::f32_add( a, float32_t{inv_float_eps} ), float32_t{inv_float_eps} ); - if (::f32_eq( y, {0} ) ) - return s ? -0.0f : 0.0f; - return from_softfloat32(y); - } - - // float relops - bool _eosio_f32_eq( float a, float b ) { return ::f32_eq( to_softfloat32(a), to_softfloat32(b) ); } - bool _eosio_f32_ne( float a, float b ) { return !::f32_eq( to_softfloat32(a), to_softfloat32(b) ); } - bool _eosio_f32_lt( float a, float b ) { return ::f32_lt( to_softfloat32(a), to_softfloat32(b) ); } - bool _eosio_f32_le( float a, float b ) { return ::f32_le( to_softfloat32(a), to_softfloat32(b) ); } - bool _eosio_f32_gt( float af, float bf ) { - float32_t a = to_softfloat32(af); - float32_t b = to_softfloat32(bf); - if (is_nan(a)) - return false; - if (is_nan(b)) - return false; - return !::f32_le( a, b ); - } - bool _eosio_f32_ge( float af, float bf ) { - float32_t a = to_softfloat32(af); - float32_t b = to_softfloat32(bf); - if (is_nan(a)) - return false; - if (is_nan(b)) - return false; - return !::f32_lt( a, b ); - } - - // double binops - double _eosio_f64_add( double a, double b ) { - float64_t ret = ::f64_add( to_softfloat64(a), to_softfloat64(b) ); - return from_softfloat64(ret); - } - double _eosio_f64_sub( double a, double b ) { - float64_t ret = ::f64_sub( to_softfloat64(a), to_softfloat64(b) ); - return from_softfloat64(ret); - } - double _eosio_f64_div( double a, double b ) { - float64_t ret = ::f64_div( to_softfloat64(a), to_softfloat64(b) ); - return from_softfloat64(ret); - } - double _eosio_f64_mul( double a, double b ) { - float64_t ret = ::f64_mul( to_softfloat64(a), to_softfloat64(b) ); - return from_softfloat64(ret); - } - double _eosio_f64_min( double af, double bf ) { - float64_t a = to_softfloat64(af); - float64_t b = to_softfloat64(bf); - if (is_nan(a)) - return af; - if (is_nan(b)) - return bf; - if (f64_sign_bit(a) != f64_sign_bit(b)) - return f64_sign_bit(a) ? af : bf; - return ::f64_lt( a, b ) ? af : bf; - } - double _eosio_f64_max( double af, double bf ) { - float64_t a = to_softfloat64(af); - float64_t b = to_softfloat64(bf); - if (is_nan(a)) - return af; - if (is_nan(b)) - return bf; - if (f64_sign_bit(a) != f64_sign_bit(b)) - return f64_sign_bit(a) ? bf : af; - return ::f64_lt( a, b ) ? bf : af; - } - double _eosio_f64_copysign( double af, double bf ) { - float64_t a = to_softfloat64(af); - float64_t b = to_softfloat64(bf); - uint64_t sign_of_b = b.v >> 63; - a.v &= ~(uint64_t(1) << 63); // clear the sign bit - a.v = a.v | (sign_of_b << 63); // add the sign of b - return from_softfloat64(a); - } - - // double unops - double _eosio_f64_abs( double af ) { - float64_t a = to_softfloat64(af); - a.v &= ~(uint64_t(1) << 63); - return from_softfloat64(a); - } - double _eosio_f64_neg( double af ) { - float64_t a = to_softfloat64(af); - uint64_t sign = a.v >> 63; - a.v &= ~(uint64_t(1) << 63); - a.v |= (uint64_t(!sign) << 63); - return from_softfloat64(a); - } - double _eosio_f64_sqrt( double a ) { - float64_t ret = ::f64_sqrt( to_softfloat64(a) ); - return from_softfloat64(ret); - } - // ceil, floor, trunc and nearest are lifted from libc - double _eosio_f64_ceil( double af ) { - float64_t a = to_softfloat64( af ); - float64_t ret; - int e = a.v >> 52 & 0x7ff; - float64_t y; - if (e >= 0x3ff+52 || ::f64_eq( a, { 0 } )) - return af; - /* y = int(x) - x, where int(x) is an integer neighbor of x */ - if (a.v >> 63) - y = ::f64_sub( ::f64_add( ::f64_sub( a, float64_t{inv_double_eps} ), float64_t{inv_double_eps} ), a ); - else - y = ::f64_sub( ::f64_sub( ::f64_add( a, float64_t{inv_double_eps} ), float64_t{inv_double_eps} ), a ); - /* special case because of non-nearest rounding modes */ - if (e <= 0x3ff-1) { - return a.v >> 63 ? -0.0 : 1.0; //float64_t{0x8000000000000000} : float64_t{0xBE99999A3F800000}; //either -0.0 or 1 - } - if (::f64_lt( y, to_softfloat64(0) )) { - ret = ::f64_add( ::f64_add( a, y ), to_softfloat64(1) ); // 0xBE99999A3F800000 } ); // plus 1 - return from_softfloat64(ret); - } - ret = ::f64_add( a, y ); - return from_softfloat64(ret); - } - double _eosio_f64_floor( double af ) { - float64_t a = to_softfloat64( af ); - float64_t ret; - int e = a.v >> 52 & 0x7FF; - float64_t y; - double de = 1/DBL_EPSILON; - if ( a.v == 0x8000000000000000) { - return af; - } - if (e >= 0x3FF+52 || a.v == 0) { - return af; - } - if (a.v >> 63) - y = ::f64_sub( ::f64_add( ::f64_sub( a, float64_t{inv_double_eps} ), float64_t{inv_double_eps} ), a ); - else - y = ::f64_sub( ::f64_sub( ::f64_add( a, float64_t{inv_double_eps} ), float64_t{inv_double_eps} ), a ); - if (e <= 0x3FF-1) { - return a.v>>63 ? -1.0 : 0.0; //float64_t{0xBFF0000000000000} : float64_t{0}; // -1 or 0 - } - if ( !::f64_le( y, float64_t{0} ) ) { - ret = ::f64_sub( ::f64_add(a,y), to_softfloat64(1.0)); - return from_softfloat64(ret); - } - ret = ::f64_add( a, y ); - return from_softfloat64(ret); - } - double _eosio_f64_trunc( double af ) { - float64_t a = to_softfloat64( af ); - int e = (int)(a.v >> 52 & 0x7ff) - 0x3ff + 12; - uint64_t m; - if (e >= 52 + 12) - return af; - if (e < 12) - e = 1; - m = -1ULL >> e; - if ((a.v & m) == 0) - return af; - a.v &= ~m; - return from_softfloat64(a); - } - - double _eosio_f64_nearest( double af ) { - float64_t a = to_softfloat64( af ); - int e = (a.v >> 52 & 0x7FF); - int s = a.v >> 63; - float64_t y; - if ( e >= 0x3FF+52 ) - return af; - if ( s ) - y = ::f64_add( ::f64_sub( a, float64_t{inv_double_eps} ), float64_t{inv_double_eps} ); - else - y = ::f64_sub( ::f64_add( a, float64_t{inv_double_eps} ), float64_t{inv_double_eps} ); - if ( ::f64_eq( y, float64_t{0} ) ) - return s ? -0.0 : 0.0; - return from_softfloat64(y); - } - - // double relops - bool _eosio_f64_eq( double a, double b ) { return ::f64_eq( to_softfloat64(a), to_softfloat64(b) ); } - bool _eosio_f64_ne( double a, double b ) { return !::f64_eq( to_softfloat64(a), to_softfloat64(b) ); } - bool _eosio_f64_lt( double a, double b ) { return ::f64_lt( to_softfloat64(a), to_softfloat64(b) ); } - bool _eosio_f64_le( double a, double b ) { return ::f64_le( to_softfloat64(a), to_softfloat64(b) ); } - bool _eosio_f64_gt( double af, double bf ) { - float64_t a = to_softfloat64(af); - float64_t b = to_softfloat64(bf); - if (is_nan(a)) - return false; - if (is_nan(b)) - return false; - return !::f64_le( a, b ); - } - bool _eosio_f64_ge( double af, double bf ) { - float64_t a = to_softfloat64(af); - float64_t b = to_softfloat64(bf); - if (is_nan(a)) - return false; - if (is_nan(b)) - return false; - return !::f64_lt( a, b ); - } - - // float and double conversions - double _eosio_f32_promote( float a ) { - return from_softfloat64(f32_to_f64( to_softfloat32(a)) ); - } - float _eosio_f64_demote( double a ) { - return from_softfloat32(f64_to_f32( to_softfloat64(a)) ); - } - int32_t _eosio_f32_trunc_i32s( float af ) { - float32_t a = to_softfloat32(af); - if (_eosio_f32_ge(af, 2147483648.0f) || _eosio_f32_lt(af, -2147483648.0f)) - FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f32.convert_s/i32 overflow" ); - - if (is_nan(a)) - FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f32.convert_s/i32 unrepresentable"); - return f32_to_i32( to_softfloat32(_eosio_f32_trunc( af )), 0, false ); - } - int32_t _eosio_f64_trunc_i32s( double af ) { - float64_t a = to_softfloat64(af); - if (_eosio_f64_ge(af, 2147483648.0) || _eosio_f64_lt(af, -2147483648.0)) - FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f64.convert_s/i32 overflow"); - if (is_nan(a)) - FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f64.convert_s/i32 unrepresentable"); - return f64_to_i32( to_softfloat64(_eosio_f64_trunc( af )), 0, false ); - } - uint32_t _eosio_f32_trunc_i32u( float af ) { - float32_t a = to_softfloat32(af); - if (_eosio_f32_ge(af, 4294967296.0f) || _eosio_f32_le(af, -1.0f)) - FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f32.convert_u/i32 overflow"); - if (is_nan(a)) - FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f32.convert_u/i32 unrepresentable"); - return f32_to_ui32( to_softfloat32(_eosio_f32_trunc( af )), 0, false ); - } - uint32_t _eosio_f64_trunc_i32u( double af ) { - float64_t a = to_softfloat64(af); - if (_eosio_f64_ge(af, 4294967296.0) || _eosio_f64_le(af, -1.0)) - FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f64.convert_u/i32 overflow"); - if (is_nan(a)) - FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f64.convert_u/i32 unrepresentable"); - return f64_to_ui32( to_softfloat64(_eosio_f64_trunc( af )), 0, false ); - } - int64_t _eosio_f32_trunc_i64s( float af ) { - float32_t a = to_softfloat32(af); - if (_eosio_f32_ge(af, 9223372036854775808.0f) || _eosio_f32_lt(af, -9223372036854775808.0f)) - FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f32.convert_s/i64 overflow"); - if (is_nan(a)) - FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f32.convert_s/i64 unrepresentable"); - return f32_to_i64( to_softfloat32(_eosio_f32_trunc( af )), 0, false ); - } - int64_t _eosio_f64_trunc_i64s( double af ) { - float64_t a = to_softfloat64(af); - if (_eosio_f64_ge(af, 9223372036854775808.0) || _eosio_f64_lt(af, -9223372036854775808.0)) - FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f64.convert_s/i64 overflow"); - if (is_nan(a)) - FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f64.convert_s/i64 unrepresentable"); - - return f64_to_i64( to_softfloat64(_eosio_f64_trunc( af )), 0, false ); - } - uint64_t _eosio_f32_trunc_i64u( float af ) { - float32_t a = to_softfloat32(af); - if (_eosio_f32_ge(af, 18446744073709551616.0f) || _eosio_f32_le(af, -1.0f)) - FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f32.convert_u/i64 overflow"); - if (is_nan(a)) - FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f32.convert_u/i64 unrepresentable"); - return f32_to_ui64( to_softfloat32(_eosio_f32_trunc( af )), 0, false ); - } - uint64_t _eosio_f64_trunc_i64u( double af ) { - float64_t a = to_softfloat64(af); - if (_eosio_f64_ge(af, 18446744073709551616.0) || _eosio_f64_le(af, -1.0)) - FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f64.convert_u/i64 overflow"); - if (is_nan(a)) - FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f64.convert_u/i64 unrepresentable"); - return f64_to_ui64( to_softfloat64(_eosio_f64_trunc( af )), 0, false ); - } - float _eosio_i32_to_f32( int32_t a ) { - return from_softfloat32(i32_to_f32( a )); - } - float _eosio_i64_to_f32( int64_t a ) { - return from_softfloat32(i64_to_f32( a )); - } - float _eosio_ui32_to_f32( uint32_t a ) { - return from_softfloat32(ui32_to_f32( a )); - } - float _eosio_ui64_to_f32( uint64_t a ) { - return from_softfloat32(ui64_to_f32( a )); - } - double _eosio_i32_to_f64( int32_t a ) { - return from_softfloat64(i32_to_f64( a )); - } - double _eosio_i64_to_f64( int64_t a ) { - return from_softfloat64(i64_to_f64( a )); - } - double _eosio_ui32_to_f64( uint32_t a ) { - return from_softfloat64(ui32_to_f64( a )); - } - double _eosio_ui64_to_f64( uint64_t a ) { - return from_softfloat64(ui64_to_f64( a )); - } - - static bool is_nan( const float32_t f ) { - return f32_is_nan( f ); - } - static bool is_nan( const float64_t f ) { - return f64_is_nan( f ); - } - static bool is_nan( const float128_t& f ) { - return f128_is_nan( f ); - } - - static constexpr uint32_t inv_float_eps = 0x4B000000; - static constexpr uint64_t inv_double_eps = 0x4330000000000000; -}; - -class producer_api : public context_aware_api { - public: - using context_aware_api::context_aware_api; - - int get_active_producers(array_ptr producers, uint32_t buffer_size) { - auto active_producers = context.get_active_producers(); - - size_t len = active_producers.size(); - auto s = len * sizeof(chain::account_name); - if( buffer_size == 0 ) return s; - - auto copy_size = std::min( static_cast(buffer_size), s ); - memcpy( producers, active_producers.data(), copy_size ); - - return copy_size; - } -}; - -class crypto_api : public context_aware_api { - public: - explicit crypto_api( apply_context& ctx ) - :context_aware_api(ctx,true){} - /** - * This method can be optimized out during replay as it has - * no possible side effects other than "passing". - */ - void assert_recover_key( const fc::sha256& digest, - array_ptr sig, uint32_t siglen, - array_ptr pub, uint32_t publen ) { - fc::crypto::signature s; - fc::crypto::public_key p; - datastream ds( sig, siglen ); - datastream pubds( pub, publen ); - - fc::raw::unpack(ds, s); - fc::raw::unpack(pubds, p); - - EOS_ASSERT(s.which() < context.db.get().num_supported_key_types, unactivated_signature_type, - "Unactivated signature type used during assert_recover_key"); - EOS_ASSERT(p.which() < context.db.get().num_supported_key_types, unactivated_key_type, - "Unactivated key type used when creating assert_recover_key"); - - if(context.control.is_producing_block()) - EOS_ASSERT(s.variable_size() <= context.control.configured_subjective_signature_length_limit(), - sig_variable_size_limit_exception, "signature variable length component size greater than subjective maximum"); - - auto check = fc::crypto::public_key( s, digest, false ); - EOS_ASSERT( check == p, crypto_api_exception, "Error expected key different than recovered key" ); - } - - int recover_key( const fc::sha256& digest, - array_ptr sig, uint32_t siglen, - array_ptr pub, uint32_t publen ) { - fc::crypto::signature s; - datastream ds( sig, siglen ); - fc::raw::unpack(ds, s); - - EOS_ASSERT(s.which() < context.db.get().num_supported_key_types, unactivated_signature_type, - "Unactivated signature type used during recover_key"); - - if(context.control.is_producing_block()) - EOS_ASSERT(s.variable_size() <= context.control.configured_subjective_signature_length_limit(), - sig_variable_size_limit_exception, "signature variable length component size greater than subjective maximum"); - - - auto recovered = fc::crypto::public_key(s, digest, false); - - // the key types newer than the first 2 may be varible in length - if (s.which() >= config::genesis_num_supported_key_types ) { - EOS_ASSERT(publen >= 33, wasm_execution_error, - "destination buffer must at least be able to hold an ECC public key"); - auto packed_pubkey = fc::raw::pack(recovered); - auto copy_size = std::min(publen, packed_pubkey.size()); - memcpy(pub, packed_pubkey.data(), copy_size); - return packed_pubkey.size(); - } else { - // legacy behavior, key types 0 and 1 always pack to 33 bytes. - // this will do one less copy for those keys while maintaining the rules of - // [0..33) dest sizes: assert (asserts in fc::raw::pack) - // [33..inf) dest sizes: return packed size (always 33) - datastream out_ds( pub, publen ); - fc::raw::pack(out_ds, recovered); - return out_ds.tellp(); - } - } - - template auto encode(char* data, uint32_t datalen) { - Encoder e; - const size_t bs = eosio::chain::config::hashing_checktime_block_size; - while ( datalen > bs ) { - e.write( data, bs ); - data += bs; - datalen -= bs; - context.trx_context.checktime(); - } - e.write( data, datalen ); - return e.result(); - } - - void assert_sha256(array_ptr data, uint32_t datalen, const fc::sha256& hash_val) { - auto result = encode( data, datalen ); - EOS_ASSERT( result == hash_val, crypto_api_exception, "hash mismatch" ); - } - - void assert_sha1(array_ptr data, uint32_t datalen, const fc::sha1& hash_val) { - auto result = encode( data, datalen ); - EOS_ASSERT( result == hash_val, crypto_api_exception, "hash mismatch" ); - } - - void assert_sha512(array_ptr data, uint32_t datalen, const fc::sha512& hash_val) { - auto result = encode( data, datalen ); - EOS_ASSERT( result == hash_val, crypto_api_exception, "hash mismatch" ); - } - - void assert_ripemd160(array_ptr data, uint32_t datalen, const fc::ripemd160& hash_val) { - auto result = encode( data, datalen ); - EOS_ASSERT( result == hash_val, crypto_api_exception, "hash mismatch" ); - } - - void sha1(array_ptr data, uint32_t datalen, fc::sha1& hash_val) { - hash_val = encode( data, datalen ); - } - - void sha256(array_ptr data, uint32_t datalen, fc::sha256& hash_val) { - hash_val = encode( data, datalen ); - } - - void sha512(array_ptr data, uint32_t datalen, fc::sha512& hash_val) { - hash_val = encode( data, datalen ); - } - - void ripemd160(array_ptr data, uint32_t datalen, fc::ripemd160& hash_val) { - hash_val = encode( data, datalen ); - } -}; - -class permission_api : public context_aware_api { - public: - using context_aware_api::context_aware_api; - - bool check_transaction_authorization( array_ptr trx_data, uint32_t trx_size, - array_ptr pubkeys_data, uint32_t pubkeys_size, - array_ptr perms_data, uint32_t perms_size - ) - { - transaction trx = fc::raw::unpack( trx_data, trx_size ); - - flat_set provided_keys; - unpack_provided_keys( provided_keys, pubkeys_data, pubkeys_size ); - - flat_set provided_permissions; - unpack_provided_permissions( provided_permissions, perms_data, perms_size ); - - try { - context.control - .get_authorization_manager() - .check_authorization( trx.actions, - provided_keys, - provided_permissions, - fc::seconds(trx.delay_sec), - std::bind(&transaction_context::checktime, &context.trx_context), - false - ); - return true; - } catch( const authorization_exception& e ) {} - - return false; - } - - bool check_permission_authorization( account_name account, permission_name permission, - array_ptr pubkeys_data, uint32_t pubkeys_size, - array_ptr perms_data, uint32_t perms_size, - uint64_t delay_us - ) - { - EOS_ASSERT( delay_us <= static_cast(std::numeric_limits::max()), - action_validate_exception, "provided delay is too large" ); - - flat_set provided_keys; - unpack_provided_keys( provided_keys, pubkeys_data, pubkeys_size ); - - flat_set provided_permissions; - unpack_provided_permissions( provided_permissions, perms_data, perms_size ); - - try { - context.control - .get_authorization_manager() - .check_authorization( account, - permission, - provided_keys, - provided_permissions, - fc::microseconds(delay_us), - std::bind(&transaction_context::checktime, &context.trx_context), - false - ); - return true; - } catch( const authorization_exception& e ) {} - - return false; - } - - int64_t get_permission_last_used( account_name account, permission_name permission ) { - const auto& am = context.control.get_authorization_manager(); - return am.get_permission_last_used( am.get_permission({account, permission}) ).time_since_epoch().count(); - }; - - int64_t get_account_creation_time( account_name account ) { - auto* acct = context.db.find(account); - EOS_ASSERT( acct != nullptr, action_validate_exception, - "account '${account}' does not exist", ("account", account) ); - return time_point(acct->creation_date).time_since_epoch().count(); - } - - private: - void unpack_provided_keys( flat_set& keys, const char* pubkeys_data, uint32_t pubkeys_size ) { - keys.clear(); - if( pubkeys_size == 0 ) return; - - keys = fc::raw::unpack>( pubkeys_data, pubkeys_size ); - } - - void unpack_provided_permissions( flat_set& permissions, const char* perms_data, uint32_t perms_size ) { - permissions.clear(); - if( perms_size == 0 ) return; - - permissions = fc::raw::unpack>( perms_data, perms_size ); - } - -}; - -class authorization_api : public context_aware_api { - public: - using context_aware_api::context_aware_api; - - void require_auth( account_name account ) { - context.require_authorization( account ); - } - - bool has_auth( account_name account )const { - return context.has_authorization( account ); - } - - void require_auth2( account_name account, - permission_name permission) { - context.require_authorization( account, permission ); - } - - void require_recipient( account_name recipient ) { - context.require_recipient( recipient ); - } - - bool is_account( account_name account )const { - return context.is_account( account ); - } - -}; - -class system_api : public context_aware_api { - public: - using context_aware_api::context_aware_api; - - uint64_t current_time() { - return static_cast( context.control.pending_block_time().time_since_epoch().count() ); - } - - uint64_t publication_time() { - return static_cast( context.trx_context.published.time_since_epoch().count() ); - } - - /** - * Returns true if the specified protocol feature is activated, false if not. - */ - bool is_feature_activated( const digest_type& feature_digest ) { - return context.control.is_protocol_feature_activated( feature_digest ); - } - - name get_sender() { - return context.get_sender(); - } -}; - -constexpr size_t max_assert_message = 1024; - -class context_free_system_api : public context_aware_api { -public: - explicit context_free_system_api( apply_context& ctx ) - :context_aware_api(ctx,true){} - - void abort() { - EOS_ASSERT( false, abort_called, "abort() called"); - } - - // Kept as intrinsic rather than implementing on WASM side (using eosio_assert_message and strlen) because strlen is faster on native side. - void eosio_assert( bool condition, null_terminated_ptr msg ) { - if( BOOST_UNLIKELY( !condition ) ) { - const size_t sz = strnlen( msg, max_assert_message ); - std::string message( msg, sz ); - EOS_THROW( eosio_assert_message_exception, "assertion failure with message: ${s}", ("s",message) ); - } - } - - void eosio_assert_message( bool condition, array_ptr msg, uint32_t msg_len ) { - if( BOOST_UNLIKELY( !condition ) ) { - const size_t sz = msg_len > max_assert_message ? max_assert_message : msg_len; - std::string message( msg, sz ); - EOS_THROW( eosio_assert_message_exception, "assertion failure with message: ${s}", ("s",message) ); - } - } - - void eosio_assert_code( bool condition, uint64_t error_code ) { - if( BOOST_UNLIKELY( !condition ) ) { - if( error_code >= static_cast(system_error_code::generic_system_error) ) { - restricted_error_code_exception e( FC_LOG_MESSAGE( - error, - "eosio_assert_code called with reserved error code: ${error_code}", - ("error_code", error_code) - ) ); - e.error_code = static_cast(system_error_code::contract_restricted_error_code); - throw e; - } else { - eosio_assert_code_exception e( FC_LOG_MESSAGE( - error, - "assertion failure with error code: ${error_code}", - ("error_code", error_code) - ) ); - e.error_code = error_code; - throw e; - } - } - } - - void eosio_exit(int32_t code) { - context.control.get_wasm_interface().exit(); - } - -}; - -class action_api : public context_aware_api { - public: - action_api( apply_context& ctx ) - :context_aware_api(ctx,true){} - - int read_action_data(array_ptr memory, uint32_t buffer_size) { - auto s = context.get_action().data.size(); - if( buffer_size == 0 ) return s; - - auto copy_size = std::min( static_cast(buffer_size), s ); - memcpy( (char*)memory.value, context.get_action().data.data(), copy_size ); - - return copy_size; - } - - int action_data_size() { - return context.get_action().data.size(); - } - - name current_receiver() { - return context.get_receiver(); - } -}; - -class console_api : public context_aware_api { - public: - console_api( apply_context& ctx ) - : context_aware_api(ctx,true) - , ignore(!ctx.control.contracts_console()) {} - - // Kept as intrinsic rather than implementing on WASM side (using prints_l and strlen) because strlen is faster on native side. - void prints(null_terminated_ptr str) { - if ( !ignore ) { - context.console_append( static_cast(str) ); - } - } - - void prints_l(array_ptr str, uint32_t str_len ) { - if ( !ignore ) { - context.console_append(string(str, str_len)); - } - } - - void printi(int64_t val) { - if ( !ignore ) { - std::ostringstream oss; - oss << val; - context.console_append( oss.str() ); - } - } - - void printui(uint64_t val) { - if ( !ignore ) { - std::ostringstream oss; - oss << val; - context.console_append( oss.str() ); - } - } - - void printi128(const __int128& val) { - if ( !ignore ) { - bool is_negative = (val < 0); - unsigned __int128 val_magnitude; - - if( is_negative ) - val_magnitude = static_cast(-val); // Works even if val is at the lowest possible value of a int128_t - else - val_magnitude = static_cast(val); - - fc::uint128_t v(val_magnitude>>64, static_cast(val_magnitude) ); - - string s; - if( is_negative ) { - s += '-'; - } - s += fc::variant(v).get_string(); - - context.console_append( s ); - } - } - - void printui128(const unsigned __int128& val) { - if ( !ignore ) { - fc::uint128_t v(val>>64, static_cast(val) ); - context.console_append(fc::variant(v).get_string()); - } - } - - void printsf( float val ) { - if ( !ignore ) { - // Assumes float representation on native side is the same as on the WASM side - std::ostringstream oss; - oss.setf( std::ios::scientific, std::ios::floatfield ); - oss.precision( std::numeric_limits::digits10 ); - oss << val; - context.console_append( oss.str() ); - } - } - - void printdf( double val ) { - if ( !ignore ) { - // Assumes double representation on native side is the same as on the WASM side - std::ostringstream oss; - oss.setf( std::ios::scientific, std::ios::floatfield ); - oss.precision( std::numeric_limits::digits10 ); - oss << val; - context.console_append( oss.str() ); - } - } - - void printqf( const float128_t& val ) { - /* - * Native-side long double uses an 80-bit extended-precision floating-point number. - * The easiest solution for now was to use the Berkeley softfloat library to round the 128-bit - * quadruple-precision floating-point number to an 80-bit extended-precision floating-point number - * (losing precision) which then allows us to simply cast it into a long double for printing purposes. - * - * Later we might find a better solution to print the full quadruple-precision floating-point number. - * Maybe with some compilation flag that turns long double into a quadruple-precision floating-point number, - * or maybe with some library that allows us to print out quadruple-precision floating-point numbers without - * having to deal with long doubles at all. - */ - - if ( !ignore ) { - std::ostringstream oss; - oss.setf( std::ios::scientific, std::ios::floatfield ); - -#ifdef __x86_64__ - oss.precision( std::numeric_limits::digits10 ); - extFloat80_t val_approx; - f128M_to_extF80M(&val, &val_approx); -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wstrict-aliasing" - oss << *(long double*)(&val_approx); -#pragma GCC diagnostic pop -#else - oss.precision( std::numeric_limits::digits10 ); - double val_approx = from_softfloat64( f128M_to_f64(&val) ); - oss << val_approx; -#endif - context.console_append( oss.str() ); - } - } - - void printn(name value) { - if ( !ignore ) { - context.console_append(value.to_string()); - } - } - - void printhex(array_ptr data, uint32_t data_len ) { - if ( !ignore ) { - context.console_append(fc::to_hex(data, data_len)); - } - } - - private: - bool ignore; -}; - -#define DB_API_METHOD_WRAPPERS_SIMPLE_SECONDARY(IDX, TYPE)\ - int db_##IDX##_store( uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, const TYPE& secondary ) {\ - return context.IDX.store( scope, table, account_name(payer), id, secondary );\ - }\ - void db_##IDX##_update( int iterator, uint64_t payer, const TYPE& secondary ) {\ - return context.IDX.update( iterator, account_name(payer), secondary );\ - }\ - void db_##IDX##_remove( int iterator ) {\ - return context.IDX.remove( iterator );\ - }\ - int db_##IDX##_find_secondary( uint64_t code, uint64_t scope, uint64_t table, const TYPE& secondary, uint64_t& primary ) {\ - return context.IDX.find_secondary(code, scope, table, secondary, primary);\ - }\ - int db_##IDX##_find_primary( uint64_t code, uint64_t scope, uint64_t table, TYPE& secondary, uint64_t primary ) {\ - return context.IDX.find_primary(code, scope, table, secondary, primary);\ - }\ - int db_##IDX##_lowerbound( uint64_t code, uint64_t scope, uint64_t table, TYPE& secondary, uint64_t& primary ) {\ - return context.IDX.lowerbound_secondary(code, scope, table, secondary, primary);\ - }\ - int db_##IDX##_upperbound( uint64_t code, uint64_t scope, uint64_t table, TYPE& secondary, uint64_t& primary ) {\ - return context.IDX.upperbound_secondary(code, scope, table, secondary, primary);\ - }\ - int db_##IDX##_end( uint64_t code, uint64_t scope, uint64_t table ) {\ - return context.IDX.end_secondary(code, scope, table);\ - }\ - int db_##IDX##_next( int iterator, uint64_t& primary ) {\ - return context.IDX.next_secondary(iterator, primary);\ - }\ - int db_##IDX##_previous( int iterator, uint64_t& primary ) {\ - return context.IDX.previous_secondary(iterator, primary);\ - } - -#define DB_API_METHOD_WRAPPERS_ARRAY_SECONDARY(IDX, ARR_SIZE, ARR_ELEMENT_TYPE)\ - int db_##IDX##_store( uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, array_ptr data, uint32_t data_len) {\ - EOS_ASSERT( data_len == ARR_SIZE,\ - db_api_exception,\ - "invalid size of secondary key array for " #IDX ": given ${given} bytes but expected ${expected} bytes",\ - ("given",data_len)("expected",ARR_SIZE) );\ - return context.IDX.store(scope, table, account_name(payer), id, data.value);\ - }\ - void db_##IDX##_update( int iterator, uint64_t payer, array_ptr data, uint32_t data_len ) {\ - EOS_ASSERT( data_len == ARR_SIZE,\ - db_api_exception,\ - "invalid size of secondary key array for " #IDX ": given ${given} bytes but expected ${expected} bytes",\ - ("given",data_len)("expected",ARR_SIZE) );\ - return context.IDX.update(iterator, account_name(payer), data.value);\ - }\ - void db_##IDX##_remove( int iterator ) {\ - return context.IDX.remove(iterator);\ - }\ - int db_##IDX##_find_secondary( uint64_t code, uint64_t scope, uint64_t table, array_ptr data, uint32_t data_len, uint64_t& primary ) {\ - EOS_ASSERT( data_len == ARR_SIZE,\ - db_api_exception,\ - "invalid size of secondary key array for " #IDX ": given ${given} bytes but expected ${expected} bytes",\ - ("given",data_len)("expected",ARR_SIZE) );\ - return context.IDX.find_secondary(code, scope, table, data, primary);\ - }\ - int db_##IDX##_find_primary( uint64_t code, uint64_t scope, uint64_t table, array_ptr data, uint32_t data_len, uint64_t primary ) {\ - EOS_ASSERT( data_len == ARR_SIZE,\ - db_api_exception,\ - "invalid size of secondary key array for " #IDX ": given ${given} bytes but expected ${expected} bytes",\ - ("given",data_len)("expected",ARR_SIZE) );\ - return context.IDX.find_primary(code, scope, table, data.value, primary);\ - }\ - int db_##IDX##_lowerbound( uint64_t code, uint64_t scope, uint64_t table, array_ptr data, uint32_t data_len, uint64_t& primary ) {\ - EOS_ASSERT( data_len == ARR_SIZE,\ - db_api_exception,\ - "invalid size of secondary key array for " #IDX ": given ${given} bytes but expected ${expected} bytes",\ - ("given",data_len)("expected",ARR_SIZE) );\ - return context.IDX.lowerbound_secondary(code, scope, table, data.value, primary);\ - }\ - int db_##IDX##_upperbound( uint64_t code, uint64_t scope, uint64_t table, array_ptr data, uint32_t data_len, uint64_t& primary ) {\ - EOS_ASSERT( data_len == ARR_SIZE,\ - db_api_exception,\ - "invalid size of secondary key array for " #IDX ": given ${given} bytes but expected ${expected} bytes",\ - ("given",data_len)("expected",ARR_SIZE) );\ - return context.IDX.upperbound_secondary(code, scope, table, data.value, primary);\ - }\ - int db_##IDX##_end( uint64_t code, uint64_t scope, uint64_t table ) {\ - return context.IDX.end_secondary(code, scope, table);\ - }\ - int db_##IDX##_next( int iterator, uint64_t& primary ) {\ - return context.IDX.next_secondary(iterator, primary);\ - }\ - int db_##IDX##_previous( int iterator, uint64_t& primary ) {\ - return context.IDX.previous_secondary(iterator, primary);\ - } - -#define DB_API_METHOD_WRAPPERS_FLOAT_SECONDARY(IDX, TYPE)\ - int db_##IDX##_store( uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, const TYPE& secondary ) {\ - EOS_ASSERT( !softfloat_api::is_nan( secondary ), transaction_exception, "NaN is not an allowed value for a secondary key" );\ - return context.IDX.store( scope, table, account_name(payer), id, secondary );\ - }\ - void db_##IDX##_update( int iterator, uint64_t payer, const TYPE& secondary ) {\ - EOS_ASSERT( !softfloat_api::is_nan( secondary ), transaction_exception, "NaN is not an allowed value for a secondary key" );\ - return context.IDX.update( iterator, account_name(payer), secondary );\ - }\ - void db_##IDX##_remove( int iterator ) {\ - return context.IDX.remove( iterator );\ - }\ - int db_##IDX##_find_secondary( uint64_t code, uint64_t scope, uint64_t table, const TYPE& secondary, uint64_t& primary ) {\ - EOS_ASSERT( !softfloat_api::is_nan( secondary ), transaction_exception, "NaN is not an allowed value for a secondary key" );\ - return context.IDX.find_secondary(code, scope, table, secondary, primary);\ - }\ - int db_##IDX##_find_primary( uint64_t code, uint64_t scope, uint64_t table, TYPE& secondary, uint64_t primary ) {\ - return context.IDX.find_primary(code, scope, table, secondary, primary);\ - }\ - int db_##IDX##_lowerbound( uint64_t code, uint64_t scope, uint64_t table, TYPE& secondary, uint64_t& primary ) {\ - EOS_ASSERT( !softfloat_api::is_nan( secondary ), transaction_exception, "NaN is not an allowed value for a secondary key" );\ - return context.IDX.lowerbound_secondary(code, scope, table, secondary, primary);\ - }\ - int db_##IDX##_upperbound( uint64_t code, uint64_t scope, uint64_t table, TYPE& secondary, uint64_t& primary ) {\ - EOS_ASSERT( !softfloat_api::is_nan( secondary ), transaction_exception, "NaN is not an allowed value for a secondary key" );\ - return context.IDX.upperbound_secondary(code, scope, table, secondary, primary);\ - }\ - int db_##IDX##_end( uint64_t code, uint64_t scope, uint64_t table ) {\ - return context.IDX.end_secondary(code, scope, table);\ - }\ - int db_##IDX##_next( int iterator, uint64_t& primary ) {\ - return context.IDX.next_secondary(iterator, primary);\ - }\ - int db_##IDX##_previous( int iterator, uint64_t& primary ) {\ - return context.IDX.previous_secondary(iterator, primary);\ - } - -class database_api : public context_aware_api { - public: - using context_aware_api::context_aware_api; - - int db_store_i64( uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, array_ptr buffer, uint32_t buffer_size ) { - return context.db_store_i64( name(scope), name(table), account_name(payer), id, buffer, buffer_size ); - } - void db_update_i64( int itr, uint64_t payer, array_ptr buffer, uint32_t buffer_size ) { - context.db_update_i64( itr, account_name(payer), buffer, buffer_size ); - } - void db_remove_i64( int itr ) { - context.db_remove_i64( itr ); - } - int db_get_i64( int itr, array_ptr buffer, uint32_t buffer_size ) { - return context.db_get_i64( itr, buffer, buffer_size ); - } - int db_next_i64( int itr, uint64_t& primary ) { - return context.db_next_i64(itr, primary); - } - int db_previous_i64( int itr, uint64_t& primary ) { - return context.db_previous_i64(itr, primary); - } - int db_find_i64( uint64_t code, uint64_t scope, uint64_t table, uint64_t id ) { - return context.db_find_i64( name(code), name(scope), name(table), id ); - } - int db_lowerbound_i64( uint64_t code, uint64_t scope, uint64_t table, uint64_t id ) { - return context.db_lowerbound_i64( name(code), name(scope), name(table), id ); - } - int db_upperbound_i64( uint64_t code, uint64_t scope, uint64_t table, uint64_t id ) { - return context.db_upperbound_i64( name(code), name(scope), name(table), id ); - } - int db_end_i64( uint64_t code, uint64_t scope, uint64_t table ) { - return context.db_end_i64( name(code), name(scope), name(table) ); - } - - DB_API_METHOD_WRAPPERS_SIMPLE_SECONDARY(idx64, uint64_t) - DB_API_METHOD_WRAPPERS_SIMPLE_SECONDARY(idx128, uint128_t) - DB_API_METHOD_WRAPPERS_ARRAY_SECONDARY(idx256, 2, uint128_t) - DB_API_METHOD_WRAPPERS_FLOAT_SECONDARY(idx_double, float64_t) - DB_API_METHOD_WRAPPERS_FLOAT_SECONDARY(idx_long_double, float128_t) -}; - -class memory_api : public context_aware_api { - public: - memory_api( apply_context& ctx ) - :context_aware_api(ctx,true){} - - char* memcpy( array_ptr dest, array_ptr src, uint32_t length) { - EOS_ASSERT((size_t)(std::abs((ptrdiff_t)dest.value - (ptrdiff_t)src.value)) >= length, - overlapping_memory_error, "memcpy can only accept non-aliasing pointers"); - return (char *)::memcpy(dest, src, length); - } - - char* memmove( array_ptr dest, array_ptr src, uint32_t length) { - return (char *)::memmove(dest, src, length); - } - - int memcmp( array_ptr dest, array_ptr src, uint32_t length) { - int ret = ::memcmp(dest, src, length); - if(ret < 0) - return -1; - if(ret > 0) - return 1; - return 0; - } - - char* memset( array_ptr dest, int value, uint32_t length ) { - return (char *)::memset( dest, value, length ); - } -}; - -class transaction_api : public context_aware_api { - public: - using context_aware_api::context_aware_api; - - void send_inline( array_ptr data, uint32_t data_len ) { - //TODO: Why is this limit even needed? And why is it not consistently checked on actions in input or deferred transactions - EOS_ASSERT( data_len < context.control.get_global_properties().configuration.max_inline_action_size, inline_action_too_big, - "inline action too big" ); - - action act; - fc::raw::unpack(data, data_len, act); - context.execute_inline(std::move(act)); - } - - void send_context_free_inline( array_ptr data, uint32_t data_len ) { - //TODO: Why is this limit even needed? And why is it not consistently checked on actions in input or deferred transactions - EOS_ASSERT( data_len < context.control.get_global_properties().configuration.max_inline_action_size, inline_action_too_big, - "inline action too big" ); - - action act; - fc::raw::unpack(data, data_len, act); - context.execute_context_free_inline(std::move(act)); - } - - void send_deferred( const uint128_t& sender_id, account_name payer, array_ptr data, uint32_t data_len, uint32_t replace_existing) { - transaction trx; - fc::raw::unpack(data, data_len, trx); - context.schedule_deferred_transaction(sender_id, payer, std::move(trx), replace_existing); - } - - bool cancel_deferred( const unsigned __int128& val ) { - fc::uint128_t sender_id(val>>64, uint64_t(val) ); - return context.cancel_deferred_transaction( (unsigned __int128)sender_id ); - } -}; - - -class context_free_transaction_api : public context_aware_api { - public: - context_free_transaction_api( apply_context& ctx ) - :context_aware_api(ctx,true){} - - int read_transaction( array_ptr data, uint32_t buffer_size ) { - bytes trx = context.get_packed_transaction(); - - auto s = trx.size(); - if( buffer_size == 0) return s; - - auto copy_size = std::min( static_cast(buffer_size), s ); - memcpy( data, trx.data(), copy_size ); - - return copy_size; - } - - int transaction_size() { - return context.get_packed_transaction().size(); - } - - int expiration() { - return context.trx_context.trx.expiration.sec_since_epoch(); - } - - int tapos_block_num() { - return context.trx_context.trx.ref_block_num; - } - int tapos_block_prefix() { - return context.trx_context.trx.ref_block_prefix; - } - - int get_action( uint32_t type, uint32_t index, array_ptr buffer, uint32_t buffer_size )const { - return context.get_action( type, index, buffer, buffer_size ); - } -}; - -class compiler_builtins : public context_aware_api { - public: - compiler_builtins( apply_context& ctx ) - :context_aware_api(ctx,true){} - - void __ashlti3(__int128& ret, uint64_t low, uint64_t high, uint32_t shift) { - fc::uint128_t i(high, low); - i <<= shift; - ret = (unsigned __int128)i; - } - - void __ashrti3(__int128& ret, uint64_t low, uint64_t high, uint32_t shift) { - // retain the signedness - ret = high; - ret <<= 64; - ret |= low; - ret >>= shift; - } - - void __lshlti3(__int128& ret, uint64_t low, uint64_t high, uint32_t shift) { - fc::uint128_t i(high, low); - i <<= shift; - ret = (unsigned __int128)i; - } - - void __lshrti3(__int128& ret, uint64_t low, uint64_t high, uint32_t shift) { - fc::uint128_t i(high, low); - i >>= shift; - ret = (unsigned __int128)i; - } - - void __divti3(__int128& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { - __int128 lhs = ha; - __int128 rhs = hb; - - lhs <<= 64; - lhs |= la; - - rhs <<= 64; - rhs |= lb; - - EOS_ASSERT(rhs != 0, arithmetic_exception, "divide by zero"); - - lhs /= rhs; - - ret = lhs; - } - - void __udivti3(unsigned __int128& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { - unsigned __int128 lhs = ha; - unsigned __int128 rhs = hb; - - lhs <<= 64; - lhs |= la; - - rhs <<= 64; - rhs |= lb; - - EOS_ASSERT(rhs != 0, arithmetic_exception, "divide by zero"); - - lhs /= rhs; - ret = lhs; - } - - void __multi3(__int128& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { - __int128 lhs = ha; - __int128 rhs = hb; - - lhs <<= 64; - lhs |= la; - - rhs <<= 64; - rhs |= lb; - - lhs *= rhs; - ret = lhs; - } - - void __modti3(__int128& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { - __int128 lhs = ha; - __int128 rhs = hb; - - lhs <<= 64; - lhs |= la; - - rhs <<= 64; - rhs |= lb; - - EOS_ASSERT(rhs != 0, arithmetic_exception, "divide by zero"); - - lhs %= rhs; - ret = lhs; - } - - void __umodti3(unsigned __int128& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { - unsigned __int128 lhs = ha; - unsigned __int128 rhs = hb; - - lhs <<= 64; - lhs |= la; - - rhs <<= 64; - rhs |= lb; - - EOS_ASSERT(rhs != 0, arithmetic_exception, "divide by zero"); - - lhs %= rhs; - ret = lhs; - } - - // arithmetic long double - void __addtf3( float128_t& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) { - float128_t a = {{ la, ha }}; - float128_t b = {{ lb, hb }}; - ret = f128_add( a, b ); - } - void __subtf3( float128_t& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) { - float128_t a = {{ la, ha }}; - float128_t b = {{ lb, hb }}; - ret = f128_sub( a, b ); - } - void __multf3( float128_t& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) { - float128_t a = {{ la, ha }}; - float128_t b = {{ lb, hb }}; - ret = f128_mul( a, b ); - } - void __divtf3( float128_t& ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) { - float128_t a = {{ la, ha }}; - float128_t b = {{ lb, hb }}; - ret = f128_div( a, b ); - } - void __negtf2( float128_t& ret, uint64_t la, uint64_t ha ) { - ret = {{ la, (ha ^ (uint64_t)1 << 63) }}; - } - - // conversion long double - void __extendsftf2( float128_t& ret, float f ) { - ret = f32_to_f128( to_softfloat32(f) ); - } - void __extenddftf2( float128_t& ret, double d ) { - ret = f64_to_f128( to_softfloat64(d) ); - } - double __trunctfdf2( uint64_t l, uint64_t h ) { - float128_t f = {{ l, h }}; - return from_softfloat64(f128_to_f64( f )); - } - float __trunctfsf2( uint64_t l, uint64_t h ) { - float128_t f = {{ l, h }}; - return from_softfloat32(f128_to_f32( f )); - } - int32_t __fixtfsi( uint64_t l, uint64_t h ) { - float128_t f = {{ l, h }}; - return f128_to_i32( f, 0, false ); - } - int64_t __fixtfdi( uint64_t l, uint64_t h ) { - float128_t f = {{ l, h }}; - return f128_to_i64( f, 0, false ); - } - void __fixtfti( __int128& ret, uint64_t l, uint64_t h ) { - float128_t f = {{ l, h }}; - ret = ___fixtfti( f ); - } - uint32_t __fixunstfsi( uint64_t l, uint64_t h ) { - float128_t f = {{ l, h }}; - return f128_to_ui32( f, 0, false ); - } - uint64_t __fixunstfdi( uint64_t l, uint64_t h ) { - float128_t f = {{ l, h }}; - return f128_to_ui64( f, 0, false ); - } - void __fixunstfti( unsigned __int128& ret, uint64_t l, uint64_t h ) { - float128_t f = {{ l, h }}; - ret = ___fixunstfti( f ); - } - void __fixsfti( __int128& ret, float a ) { - ret = ___fixsfti( to_softfloat32(a).v ); - } - void __fixdfti( __int128& ret, double a ) { - ret = ___fixdfti( to_softfloat64(a).v ); - } - void __fixunssfti( unsigned __int128& ret, float a ) { - ret = ___fixunssfti( to_softfloat32(a).v ); - } - void __fixunsdfti( unsigned __int128& ret, double a ) { - ret = ___fixunsdfti( to_softfloat64(a).v ); - } - double __floatsidf( int32_t i ) { - return from_softfloat64(i32_to_f64(i)); - } - void __floatsitf( float128_t& ret, int32_t i ) { - ret = i32_to_f128(i); - } - void __floatditf( float128_t& ret, uint64_t a ) { - ret = i64_to_f128( a ); - } - void __floatunsitf( float128_t& ret, uint32_t i ) { - ret = ui32_to_f128(i); - } - void __floatunditf( float128_t& ret, uint64_t a ) { - ret = ui64_to_f128( a ); - } - double __floattidf( uint64_t l, uint64_t h ) { - fc::uint128_t v(h, l); - unsigned __int128 val = (unsigned __int128)v; - return ___floattidf( *(__int128*)&val ); - } - double __floatuntidf( uint64_t l, uint64_t h ) { - fc::uint128_t v(h, l); - return ___floatuntidf( (unsigned __int128)v ); - } - int ___cmptf2( uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb, int return_value_if_nan ) { - float128_t a = {{ la, ha }}; - float128_t b = {{ lb, hb }}; - if ( __unordtf2(la, ha, lb, hb) ) - return return_value_if_nan; - if ( f128_lt( a, b ) ) - return -1; - if ( f128_eq( a, b ) ) - return 0; - return 1; - } - int __eqtf2( uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) { - return ___cmptf2(la, ha, lb, hb, 1); - } - int __netf2( uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) { - return ___cmptf2(la, ha, lb, hb, 1); - } - int __getf2( uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) { - return ___cmptf2(la, ha, lb, hb, -1); - } - int __gttf2( uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) { - return ___cmptf2(la, ha, lb, hb, 0); - } - int __letf2( uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) { - return ___cmptf2(la, ha, lb, hb, 1); - } - int __lttf2( uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) { - return ___cmptf2(la, ha, lb, hb, 0); - } - int __cmptf2( uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) { - return ___cmptf2(la, ha, lb, hb, 1); - } - int __unordtf2( uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) { - float128_t a = {{ la, ha }}; - float128_t b = {{ lb, hb }}; - if ( softfloat_api::is_nan(a) || softfloat_api::is_nan(b) ) - return 1; - return 0; - } - - static constexpr uint32_t SHIFT_WIDTH = (sizeof(uint64_t)*8)-1; -}; - - -/* - * This api will be removed with fix for `eos #2561` - */ -class call_depth_api : public context_aware_api { - public: - call_depth_api( apply_context& ctx ) - :context_aware_api(ctx,true){} - void call_depth_assert() { - FC_THROW_EXCEPTION(wasm_execution_error, "Exceeded call depth maximum"); - } -}; - -REGISTER_INJECTED_INTRINSICS(call_depth_api, - (call_depth_assert, void() ) -); - -REGISTER_INTRINSICS(compiler_builtins, - (__ashlti3, void(int, int64_t, int64_t, int) ) - (__ashrti3, void(int, int64_t, int64_t, int) ) - (__lshlti3, void(int, int64_t, int64_t, int) ) - (__lshrti3, void(int, int64_t, int64_t, int) ) - (__divti3, void(int, int64_t, int64_t, int64_t, int64_t) ) - (__udivti3, void(int, int64_t, int64_t, int64_t, int64_t) ) - (__modti3, void(int, int64_t, int64_t, int64_t, int64_t) ) - (__umodti3, void(int, int64_t, int64_t, int64_t, int64_t) ) - (__multi3, void(int, int64_t, int64_t, int64_t, int64_t) ) - (__addtf3, void(int, int64_t, int64_t, int64_t, int64_t) ) - (__subtf3, void(int, int64_t, int64_t, int64_t, int64_t) ) - (__multf3, void(int, int64_t, int64_t, int64_t, int64_t) ) - (__divtf3, void(int, int64_t, int64_t, int64_t, int64_t) ) - (__eqtf2, int(int64_t, int64_t, int64_t, int64_t) ) - (__netf2, int(int64_t, int64_t, int64_t, int64_t) ) - (__getf2, int(int64_t, int64_t, int64_t, int64_t) ) - (__gttf2, int(int64_t, int64_t, int64_t, int64_t) ) - (__lttf2, int(int64_t, int64_t, int64_t, int64_t) ) - (__letf2, int(int64_t, int64_t, int64_t, int64_t) ) - (__cmptf2, int(int64_t, int64_t, int64_t, int64_t) ) - (__unordtf2, int(int64_t, int64_t, int64_t, int64_t) ) - (__negtf2, void (int, int64_t, int64_t) ) - (__floatsitf, void (int, int) ) - (__floatunsitf, void (int, int) ) - (__floatditf, void (int, int64_t) ) - (__floatunditf, void (int, int64_t) ) - (__floattidf, double (int64_t, int64_t) ) - (__floatuntidf, double (int64_t, int64_t) ) - (__floatsidf, double(int) ) - (__extendsftf2, void(int, float) ) - (__extenddftf2, void(int, double) ) - (__fixtfti, void(int, int64_t, int64_t) ) - (__fixtfdi, int64_t(int64_t, int64_t) ) - (__fixtfsi, int(int64_t, int64_t) ) - (__fixunstfti, void(int, int64_t, int64_t) ) - (__fixunstfdi, int64_t(int64_t, int64_t) ) - (__fixunstfsi, int(int64_t, int64_t) ) - (__fixsfti, void(int, float) ) - (__fixdfti, void(int, double) ) - (__fixunssfti, void(int, float) ) - (__fixunsdfti, void(int, double) ) - (__trunctfdf2, double(int64_t, int64_t) ) - (__trunctfsf2, float(int64_t, int64_t) ) -); - -REGISTER_INTRINSICS(privileged_api, - (is_feature_active, int(int64_t) ) - (activate_feature, void(int64_t) ) - (get_resource_limits, void(int64_t,int,int,int) ) - (set_resource_limits, void(int64_t,int64_t,int64_t,int64_t) ) - (set_proposed_producers, int64_t(int,int) ) - (set_proposed_producers_ex, int64_t(int64_t, int, int) ) - (get_blockchain_parameters_packed, int(int, int) ) - (set_blockchain_parameters_packed, void(int,int) ) - (is_privileged, int(int64_t) ) - (set_privileged, void(int64_t, int) ) - (preactivate_feature, void(int) ) -); - -REGISTER_INJECTED_INTRINSICS(transaction_context, - (checktime, void() ) -); - -REGISTER_INTRINSICS(producer_api, - (get_active_producers, int(int, int) ) -); - -#define DB_SECONDARY_INDEX_METHODS_SIMPLE(IDX) \ - (db_##IDX##_store, int(int64_t,int64_t,int64_t,int64_t,int) )\ - (db_##IDX##_remove, void(int) )\ - (db_##IDX##_update, void(int,int64_t,int) )\ - (db_##IDX##_find_primary, int(int64_t,int64_t,int64_t,int,int64_t) )\ - (db_##IDX##_find_secondary, int(int64_t,int64_t,int64_t,int,int) )\ - (db_##IDX##_lowerbound, int(int64_t,int64_t,int64_t,int,int) )\ - (db_##IDX##_upperbound, int(int64_t,int64_t,int64_t,int,int) )\ - (db_##IDX##_end, int(int64_t,int64_t,int64_t) )\ - (db_##IDX##_next, int(int, int) )\ - (db_##IDX##_previous, int(int, int) ) - -#define DB_SECONDARY_INDEX_METHODS_ARRAY(IDX) \ - (db_##IDX##_store, int(int64_t,int64_t,int64_t,int64_t,int,int) )\ - (db_##IDX##_remove, void(int) )\ - (db_##IDX##_update, void(int,int64_t,int,int) )\ - (db_##IDX##_find_primary, int(int64_t,int64_t,int64_t,int,int,int64_t) )\ - (db_##IDX##_find_secondary, int(int64_t,int64_t,int64_t,int,int,int) )\ - (db_##IDX##_lowerbound, int(int64_t,int64_t,int64_t,int,int,int) )\ - (db_##IDX##_upperbound, int(int64_t,int64_t,int64_t,int,int,int) )\ - (db_##IDX##_end, int(int64_t,int64_t,int64_t) )\ - (db_##IDX##_next, int(int, int) )\ - (db_##IDX##_previous, int(int, int) ) - -REGISTER_INTRINSICS( database_api, - (db_store_i64, int(int64_t,int64_t,int64_t,int64_t,int,int) ) - (db_update_i64, void(int,int64_t,int,int) ) - (db_remove_i64, void(int) ) - (db_get_i64, int(int, int, int) ) - (db_next_i64, int(int, int) ) - (db_previous_i64, int(int, int) ) - (db_find_i64, int(int64_t,int64_t,int64_t,int64_t) ) - (db_lowerbound_i64, int(int64_t,int64_t,int64_t,int64_t) ) - (db_upperbound_i64, int(int64_t,int64_t,int64_t,int64_t) ) - (db_end_i64, int(int64_t,int64_t,int64_t) ) - - DB_SECONDARY_INDEX_METHODS_SIMPLE(idx64) - DB_SECONDARY_INDEX_METHODS_SIMPLE(idx128) - DB_SECONDARY_INDEX_METHODS_ARRAY(idx256) - DB_SECONDARY_INDEX_METHODS_SIMPLE(idx_double) - DB_SECONDARY_INDEX_METHODS_SIMPLE(idx_long_double) -); - -REGISTER_INTRINSICS(crypto_api, - (assert_recover_key, void(int, int, int, int, int) ) - (recover_key, int(int, int, int, int, int) ) - (assert_sha256, void(int, int, int) ) - (assert_sha1, void(int, int, int) ) - (assert_sha512, void(int, int, int) ) - (assert_ripemd160, void(int, int, int) ) - (sha1, void(int, int, int) ) - (sha256, void(int, int, int) ) - (sha512, void(int, int, int) ) - (ripemd160, void(int, int, int) ) -); - - -REGISTER_INTRINSICS(permission_api, - (check_transaction_authorization, int(int, int, int, int, int, int) ) - (check_permission_authorization, int(int64_t, int64_t, int, int, int, int, int64_t) ) - (get_permission_last_used, int64_t(int64_t, int64_t) ) - (get_account_creation_time, int64_t(int64_t) ) -); - - -REGISTER_INTRINSICS(system_api, - (current_time, int64_t() ) - (publication_time, int64_t() ) - (is_feature_activated, int(int) ) - (get_sender, int64_t() ) -); - -REGISTER_INTRINSICS(context_free_system_api, - (abort, void() ) - (eosio_assert, void(int, int) ) - (eosio_assert_message, void(int, int, int) ) - (eosio_assert_code, void(int, int64_t) ) - (eosio_exit, void(int) ) -); - -REGISTER_INTRINSICS(action_api, - (read_action_data, int(int, int) ) - (action_data_size, int() ) - (current_receiver, int64_t() ) -); - -REGISTER_INTRINSICS(authorization_api, - (require_recipient, void(int64_t) ) - (require_auth, void(int64_t) ) - (require_auth2, void(int64_t, int64_t) ) - (has_auth, int(int64_t) ) - (is_account, int(int64_t) ) -); - -REGISTER_INTRINSICS(console_api, - (prints, void(int) ) - (prints_l, void(int, int) ) - (printi, void(int64_t) ) - (printui, void(int64_t) ) - (printi128, void(int) ) - (printui128, void(int) ) - (printsf, void(float) ) - (printdf, void(double) ) - (printqf, void(int) ) - (printn, void(int64_t) ) - (printhex, void(int, int) ) -); - -REGISTER_INTRINSICS(context_free_transaction_api, - (read_transaction, int(int, int) ) - (transaction_size, int() ) - (expiration, int() ) - (tapos_block_prefix, int() ) - (tapos_block_num, int() ) - (get_action, int(int, int, int, int) ) -); - -REGISTER_INTRINSICS(transaction_api, - (send_inline, void(int, int) ) - (send_context_free_inline, void(int, int) ) - (send_deferred, void(int, int64_t, int, int, int32_t) ) - (cancel_deferred, int(int) ) -); - -REGISTER_INTRINSICS(context_free_api, - (get_context_free_data, int(int, int, int) ) -) - -REGISTER_INTRINSICS(memory_api, - (memcpy, int(int, int, int) ) - (memmove, int(int, int, int) ) - (memcmp, int(int, int, int) ) - (memset, int(int, int, int) ) -); - -REGISTER_INJECTED_INTRINSICS(softfloat_api, - (_eosio_f32_add, float(float, float) ) - (_eosio_f32_sub, float(float, float) ) - (_eosio_f32_mul, float(float, float) ) - (_eosio_f32_div, float(float, float) ) - (_eosio_f32_min, float(float, float) ) - (_eosio_f32_max, float(float, float) ) - (_eosio_f32_copysign, float(float, float) ) - (_eosio_f32_abs, float(float) ) - (_eosio_f32_neg, float(float) ) - (_eosio_f32_sqrt, float(float) ) - (_eosio_f32_ceil, float(float) ) - (_eosio_f32_floor, float(float) ) - (_eosio_f32_trunc, float(float) ) - (_eosio_f32_nearest, float(float) ) - (_eosio_f32_eq, int(float, float) ) - (_eosio_f32_ne, int(float, float) ) - (_eosio_f32_lt, int(float, float) ) - (_eosio_f32_le, int(float, float) ) - (_eosio_f32_gt, int(float, float) ) - (_eosio_f32_ge, int(float, float) ) - (_eosio_f64_add, double(double, double) ) - (_eosio_f64_sub, double(double, double) ) - (_eosio_f64_mul, double(double, double) ) - (_eosio_f64_div, double(double, double) ) - (_eosio_f64_min, double(double, double) ) - (_eosio_f64_max, double(double, double) ) - (_eosio_f64_copysign, double(double, double) ) - (_eosio_f64_abs, double(double) ) - (_eosio_f64_neg, double(double) ) - (_eosio_f64_sqrt, double(double) ) - (_eosio_f64_ceil, double(double) ) - (_eosio_f64_floor, double(double) ) - (_eosio_f64_trunc, double(double) ) - (_eosio_f64_nearest, double(double) ) - (_eosio_f64_eq, int(double, double) ) - (_eosio_f64_ne, int(double, double) ) - (_eosio_f64_lt, int(double, double) ) - (_eosio_f64_le, int(double, double) ) - (_eosio_f64_gt, int(double, double) ) - (_eosio_f64_ge, int(double, double) ) - (_eosio_f32_promote, double(float) ) - (_eosio_f64_demote, float(double) ) - (_eosio_f32_trunc_i32s, int(float) ) - (_eosio_f64_trunc_i32s, int(double) ) - (_eosio_f32_trunc_i32u, int(float) ) - (_eosio_f64_trunc_i32u, int(double) ) - (_eosio_f32_trunc_i64s, int64_t(float) ) - (_eosio_f64_trunc_i64s, int64_t(double) ) - (_eosio_f32_trunc_i64u, int64_t(float) ) - (_eosio_f64_trunc_i64u, int64_t(double) ) - (_eosio_i32_to_f32, float(int32_t) ) - (_eosio_i64_to_f32, float(int64_t) ) - (_eosio_ui32_to_f32, float(int32_t) ) - (_eosio_ui64_to_f32, float(int64_t) ) - (_eosio_i32_to_f64, double(int32_t) ) - (_eosio_i64_to_f64, double(int64_t) ) - (_eosio_ui32_to_f64, double(int32_t) ) - (_eosio_ui64_to_f64, double(int64_t) ) -); - std::istream& operator>>(std::istream& in, wasm_interface::vm_type& runtime) { std::string s; in >> s; - if (s == "wabt") - runtime = eosio::chain::wasm_interface::vm_type::wabt; - else if (s == "eos-vm") + if (s == "eos-vm") runtime = eosio::chain::wasm_interface::vm_type::eos_vm; else if (s == "eos-vm-jit") runtime = eosio::chain::wasm_interface::vm_type::eos_vm_jit; diff --git a/libraries/chain/webassembly/action.cpp b/libraries/chain/webassembly/action.cpp new file mode 100644 index 00000000000..20a914556ef --- /dev/null +++ b/libraries/chain/webassembly/action.cpp @@ -0,0 +1,32 @@ +#include +#include +#include + +namespace eosio { namespace chain { namespace webassembly { + int32_t interface::read_action_data(legacy_span memory) const { + auto s = context.get_action().data.size(); + if( memory.size() == 0 ) return s; + + auto copy_size = std::min( static_cast(memory.size()), s ); + std::memcpy( memory.data(), context.get_action().data.data(), copy_size ); + + return copy_size; + } + + int32_t interface::action_data_size() const { + return context.get_action().data.size(); + } + + name interface::current_receiver() const { + return context.get_receiver(); + } + + void interface::set_action_return_value( span packed_blob ) { + auto max_action_return_value_size = + context.control.get_global_properties().configuration.max_action_return_value_size; + EOS_ASSERT(packed_blob.size() <= max_action_return_value_size, + action_return_value_exception, + "action return value size must be less or equal to ${s} bytes", ("s", max_action_return_value_size)); + context.action_return_value.assign( packed_blob.data(), packed_blob.data() + packed_blob.size() ); + } +}}} // ns eosio::chain::webassembly diff --git a/libraries/chain/webassembly/authorization.cpp b/libraries/chain/webassembly/authorization.cpp new file mode 100644 index 00000000000..ff700583878 --- /dev/null +++ b/libraries/chain/webassembly/authorization.cpp @@ -0,0 +1,25 @@ +#include +#include + +namespace eosio { namespace chain { namespace webassembly { + void interface::require_auth( account_name account ) const { + context.require_authorization( account ); + } + + bool interface::has_auth( account_name account ) const { + return context.has_authorization( account ); + } + + void interface::require_auth2( account_name account, + permission_name permission ) const { + context.require_authorization( account, permission ); + } + + void interface::require_recipient( account_name recipient ) { + context.require_recipient( recipient ); + } + + bool interface::is_account( account_name account ) const { + return context.is_account( account ); + } +}}} // ns eosio::chain::webassembly diff --git a/libraries/chain/webassembly/cf_system.cpp b/libraries/chain/webassembly/cf_system.cpp new file mode 100644 index 00000000000..32afe936127 --- /dev/null +++ b/libraries/chain/webassembly/cf_system.cpp @@ -0,0 +1,51 @@ +#include +#include + +namespace eosio { namespace chain { namespace webassembly { + inline static constexpr size_t max_assert_message = 1024; + void interface::abort() const { + EOS_ASSERT( false, abort_called, "abort() called" ); + } + + void interface::eosio_assert( bool condition, null_terminated_ptr msg ) const { + if( BOOST_UNLIKELY( !condition ) ) { + const size_t sz = strnlen( msg.data(), max_assert_message ); + std::string message( msg.data(), sz ); + EOS_THROW( eosio_assert_message_exception, "assertion failure with message: ${s}", ("s",message) ); + } + } + + void interface::eosio_assert_message( bool condition, legacy_span msg ) const { + if( BOOST_UNLIKELY( !condition ) ) { + const size_t sz = msg.size() > max_assert_message ? max_assert_message : msg.size(); + std::string message( msg.data(), sz ); + EOS_THROW( eosio_assert_message_exception, "assertion failure with message: ${s}", ("s",message) ); + } + } + + void interface::eosio_assert_code( bool condition, uint64_t error_code ) const { + if( BOOST_UNLIKELY( !condition ) ) { + if( error_code >= static_cast(system_error_code::generic_system_error) ) { + restricted_error_code_exception e( FC_LOG_MESSAGE( + error, + "eosio_assert_code called with reserved error code: ${error_code}", + ("error_code", error_code) + ) ); + e.error_code = static_cast(system_error_code::contract_restricted_error_code); + throw e; + } else { + eosio_assert_code_exception e( FC_LOG_MESSAGE( + error, + "assertion failure with error code: ${error_code}", + ("error_code", error_code) + ) ); + e.error_code = error_code; + throw e; + } + } + } + + void interface::eosio_exit( int32_t code ) const { + context.control.get_wasm_interface().exit(); + } +}}} // ns eosio::chain::webassembly diff --git a/libraries/chain/webassembly/cf_transaction.cpp b/libraries/chain/webassembly/cf_transaction.cpp new file mode 100644 index 00000000000..647e543814a --- /dev/null +++ b/libraries/chain/webassembly/cf_transaction.cpp @@ -0,0 +1,39 @@ +#include +#include +#include + +namespace eosio { namespace chain { namespace webassembly { + int32_t interface::read_transaction( legacy_span data ) const { + if( data.size() == 0 ) return transaction_size(); + + // always pack the transaction here as exact pack format is part of consensus + // and an alternative packed format could be stored in get_packed_transaction() + const packed_transaction& packed_trx = context.trx_context.packed_trx; + bytes trx = fc::raw::pack( static_cast( packed_trx.get_transaction() ) ); + size_t copy_size = std::min( static_cast(data.size()), trx.size() ); + std::memcpy( data.data(), trx.data(), copy_size ); + + return copy_size; + } + + int32_t interface::transaction_size() const { + const packed_transaction& packed_trx = context.trx_context.packed_trx; + return fc::raw::pack_size( static_cast( packed_trx.get_transaction() ) ); + } + + int32_t interface::expiration() const { + return context.trx_context.packed_trx.get_transaction().expiration.sec_since_epoch(); + } + + int32_t interface::tapos_block_num() const { + return context.trx_context.packed_trx.get_transaction().ref_block_num; + } + + int32_t interface::tapos_block_prefix() const { + return context.trx_context.packed_trx.get_transaction().ref_block_prefix; + } + + int32_t interface::get_action( uint32_t type, uint32_t index, legacy_span buffer ) const { + return context.get_action( type, index, buffer.data(), buffer.size() ); + } +}}} // ns eosio::chain::webassembly diff --git a/libraries/chain/webassembly/compiler_builtins.cpp b/libraries/chain/webassembly/compiler_builtins.cpp new file mode 100644 index 00000000000..01f11809497 --- /dev/null +++ b/libraries/chain/webassembly/compiler_builtins.cpp @@ -0,0 +1,255 @@ +#include + +#include +#include + +#include + +namespace eosio { namespace chain { namespace webassembly { + + void interface::__ashlti3(legacy_ptr<__int128> ret, uint64_t low, uint64_t high, uint32_t shift) const { + fc::uint128 i(high, low); + i <<= shift; + *ret = (unsigned __int128)i; + } + + void interface::__ashrti3(legacy_ptr<__int128> ret, uint64_t low, uint64_t high, uint32_t shift) const { + // retain the signedness + *ret = high; + *ret <<= 64; + *ret |= low; + *ret >>= shift; + } + + void interface::__lshlti3(legacy_ptr<__int128> ret, uint64_t low, uint64_t high, uint32_t shift) const { + fc::uint128 i(high, low); + i <<= shift; + *ret = (unsigned __int128)i; + } + + void interface::__lshrti3(legacy_ptr<__int128> ret, uint64_t low, uint64_t high, uint32_t shift) const { + fc::uint128 i(high, low); + i >>= shift; + *ret = (unsigned __int128)i; + } + + void interface::__divti3(legacy_ptr<__int128> ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) const { + __int128 lhs = ha; + __int128 rhs = hb; + + lhs <<= 64; + lhs |= la; + + rhs <<= 64; + rhs |= lb; + + EOS_ASSERT(rhs != 0, arithmetic_exception, "divide by zero"); + + lhs /= rhs; + + *ret = lhs; + } + + void interface::__udivti3(legacy_ptr ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) const { + unsigned __int128 lhs = ha; + unsigned __int128 rhs = hb; + + lhs <<= 64; + lhs |= la; + + rhs <<= 64; + rhs |= lb; + + EOS_ASSERT(rhs != 0, arithmetic_exception, "divide by zero"); + + lhs /= rhs; + *ret = lhs; + } + + void interface::__multi3(legacy_ptr<__int128> ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) const { + __int128 lhs = ha; + __int128 rhs = hb; + + lhs <<= 64; + lhs |= la; + + rhs <<= 64; + rhs |= lb; + + lhs *= rhs; + *ret = lhs; + } + + void interface::__modti3(legacy_ptr<__int128> ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) const { + __int128 lhs = ha; + __int128 rhs = hb; + + lhs <<= 64; + lhs |= la; + + rhs <<= 64; + rhs |= lb; + + EOS_ASSERT(rhs != 0, arithmetic_exception, "divide by zero"); + + lhs %= rhs; + *ret = lhs; + } + + void interface::__umodti3(legacy_ptr ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) const { + unsigned __int128 lhs = ha; + unsigned __int128 rhs = hb; + + lhs <<= 64; + lhs |= la; + + rhs <<= 64; + rhs |= lb; + + EOS_ASSERT(rhs != 0, arithmetic_exception, "divide by zero"); + + lhs %= rhs; + *ret = lhs; + } + + // arithmetic long double + void interface::__addtf3( legacy_ptr ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) const { + float128_t a = {{ la, ha }}; + float128_t b = {{ lb, hb }}; + *ret = f128_add( a, b ); + } + void interface::__subtf3( legacy_ptr ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) const { + float128_t a = {{ la, ha }}; + float128_t b = {{ lb, hb }}; + *ret = f128_sub( a, b ); + } + void interface::__multf3( legacy_ptr ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) const { + float128_t a = {{ la, ha }}; + float128_t b = {{ lb, hb }}; + *ret = f128_mul( a, b ); + } + void interface::__divtf3( legacy_ptr ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) const { + float128_t a = {{ la, ha }}; + float128_t b = {{ lb, hb }}; + *ret = f128_div( a, b ); + } + void interface::__negtf2( legacy_ptr ret, uint64_t la, uint64_t ha ) const { + *ret = {{ la, (ha ^ (uint64_t)1 << 63) }}; + } + + // conversion long double + void interface::__extendsftf2( legacy_ptr ret, float f ) const { + *ret = f32_to_f128( to_softfloat32(f) ); + } + void interface::__extenddftf2( legacy_ptr ret, double d ) const { + *ret = f64_to_f128( to_softfloat64(d) ); + } + double interface::__trunctfdf2( uint64_t l, uint64_t h ) const { + float128_t f = {{ l, h }}; + return from_softfloat64(f128_to_f64( f )); + } + float interface::__trunctfsf2( uint64_t l, uint64_t h ) const { + float128_t f = {{ l, h }}; + return from_softfloat32(f128_to_f32( f )); + } + int32_t interface::__fixtfsi( uint64_t l, uint64_t h ) const { + float128_t f = {{ l, h }}; + return f128_to_i32( f, 0, false ); + } + int64_t interface::__fixtfdi( uint64_t l, uint64_t h ) const { + float128_t f = {{ l, h }}; + return f128_to_i64( f, 0, false ); + } + void interface::__fixtfti( legacy_ptr<__int128> ret, uint64_t l, uint64_t h ) const { + float128_t f = {{ l, h }}; + *ret = ___fixtfti( f ); + } + uint32_t interface::__fixunstfsi( uint64_t l, uint64_t h ) const { + float128_t f = {{ l, h }}; + return f128_to_ui32( f, 0, false ); + } + uint64_t interface::__fixunstfdi( uint64_t l, uint64_t h ) const { + float128_t f = {{ l, h }}; + return f128_to_ui64( f, 0, false ); + } + void interface::__fixunstfti( legacy_ptr ret, uint64_t l, uint64_t h ) const { + float128_t f = {{ l, h }}; + *ret = ___fixunstfti( f ); + } + void interface::__fixsfti( legacy_ptr<__int128> ret, float a ) const { + *ret = ___fixsfti( to_softfloat32(a).v ); + } + void interface::__fixdfti( legacy_ptr<__int128> ret, double a ) const { + *ret = ___fixdfti( to_softfloat64(a).v ); + } + void interface::__fixunssfti( legacy_ptr ret, float a ) const { + *ret = ___fixunssfti( to_softfloat32(a).v ); + } + void interface::__fixunsdfti( legacy_ptr ret, double a ) const { + *ret = ___fixunsdfti( to_softfloat64(a).v ); + } + double interface::__floatsidf( int32_t i ) const { + return from_softfloat64(i32_to_f64(i)); + } + void interface::__floatsitf( legacy_ptr ret, int32_t i ) const { + *ret = i32_to_f128(i); + } + void interface::__floatditf( legacy_ptr ret, uint64_t a ) const { + *ret = i64_to_f128( a ); + } + void interface::__floatunsitf( legacy_ptr ret, uint32_t i ) const { + *ret = ui32_to_f128(i); + } + void interface::__floatunditf( legacy_ptr ret, uint64_t a ) const { + *ret = ui64_to_f128( a ); + } + double interface::__floattidf( uint64_t l, uint64_t h ) const { + fc::uint128 v(h, l); + unsigned __int128 val = (unsigned __int128)v; + return ___floattidf( *(__int128*)&val ); + } + double interface::__floatuntidf( uint64_t l, uint64_t h ) const { + fc::uint128 v(h, l); + return ___floatuntidf( (unsigned __int128)v ); + } + + inline static int cmptf2_impl( const interface& i, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb, int return_value_if_nan ) { + float128_t a = {{ la, ha }}; + float128_t b = {{ lb, hb }}; + if ( i.__unordtf2(la, ha, lb, hb) ) + return return_value_if_nan; + if ( f128_lt( a, b ) ) + return -1; + if ( f128_eq( a, b ) ) + return 0; + return 1; + } + int interface::__eqtf2( uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) const { + return cmptf2_impl(*this, la, ha, lb, hb, 1); + } + int interface::__netf2( uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) const { + return cmptf2_impl(*this, la, ha, lb, hb, 1); + } + int interface::__getf2( uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) const { + return cmptf2_impl(*this, la, ha, lb, hb, -1); + } + int interface::__gttf2( uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) const { + return cmptf2_impl(*this, la, ha, lb, hb, 0); + } + int interface::__letf2( uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) const { + return cmptf2_impl(*this, la, ha, lb, hb, 1); + } + int interface::__lttf2( uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) const { + return cmptf2_impl(*this, la, ha, lb, hb, 0); + } + int interface::__cmptf2( uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) const { + return cmptf2_impl(*this, la, ha, lb, hb, 1); + } + int interface::__unordtf2( uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb ) const { + float128_t a = {{ la, ha }}; + float128_t b = {{ lb, hb }}; + if ( f128_is_nan(a) || f128_is_nan(b) ) + return 1; + return 0; + } +}}} // ns eosio::chain::webassembly diff --git a/libraries/chain/webassembly/console.cpp b/libraries/chain/webassembly/console.cpp new file mode 100644 index 00000000000..4dc7933e518 --- /dev/null +++ b/libraries/chain/webassembly/console.cpp @@ -0,0 +1,139 @@ +#include +#include +#include + +namespace eosio { namespace chain { namespace webassembly { + + template + inline static void predicated_print(Ctx& context, F&& print_func) { + if (UNLIKELY(context.control.contracts_console())) + print_func(); + } + + // Kept as intrinsic rather than implementing on WASM side (using prints_l and strlen) because strlen is faster on native side. + // TODO predicate these for ignore + void interface::prints(null_terminated_ptr str) { + predicated_print(context, + [&]() { context.console_append( static_cast(str.data()) ); }); + } + + void interface::prints_l(legacy_span str ) { + predicated_print(context, + [&]() { context.console_append(std::string_view(str.data(), str.size())); }); + } + + void interface::printi(int64_t val) { + predicated_print(context, + [&]() { + std::ostringstream oss; + oss << val; + context.console_append( oss.str() ); + }); + } + + void interface::printui(uint64_t val) { + predicated_print(context, + [&]() { + std::ostringstream oss; + oss << val; + context.console_append( oss.str() ); + }); + } + + void interface::printi128(legacy_ptr val) { + predicated_print(context, + [&]() { + bool is_negative = (*val < 0); + unsigned __int128 val_magnitude; + + if( is_negative ) + val_magnitude = static_cast(-*val); // Works even if val is at the lowest possible value of a int128_t + else + val_magnitude = static_cast(*val); + + fc::uint128 v(val_magnitude>>64, static_cast(val_magnitude) ); + + string s; + if( is_negative ) { + s += '-'; + } + s += fc::variant(v).get_string(); + + context.console_append( s ); + }); + } + + void interface::printui128(legacy_ptr val) { + predicated_print(context, + [&]() { + fc::uint128 v(*val>>64, static_cast(*val) ); + context.console_append(fc::variant(v).get_string()); + }); + } + + void interface::printsf( float32_t val ) { + predicated_print(context, + [&]() { + // Assumes float representation on native side is the same as on the WASM side + std::ostringstream oss; + oss.setf( std::ios::scientific, std::ios::floatfield ); + oss.precision( std::numeric_limits::digits10 ); + oss << ::from_softfloat32(val); + context.console_append( oss.str() ); + }); + } + + void interface::printdf( float64_t val ) { + predicated_print(context, + [&]() { + // Assumes double representation on native side is the same as on the WASM side + std::ostringstream oss; + oss.setf( std::ios::scientific, std::ios::floatfield ); + oss.precision( std::numeric_limits::digits10 ); + oss << ::from_softfloat64(val); + context.console_append( oss.str() ); + }); + } + + void interface::printqf( legacy_ptr val ) { + /* + * Native-side long double uses an 80-bit extended-precision floating-point number. + * The easiest solution for now was to use the Berkeley softfloat library to round the 128-bit + * quadruple-precision floating-point number to an 80-bit extended-precision floating-point number + * (losing precision) which then allows us to simply cast it into a long double for printing purposes. + * + * Later we might find a better solution to print the full quadruple-precision floating-point number. + * Maybe with some compilation flag that turns long double into a quadruple-precision floating-point number, + * or maybe with some library that allows us to print out quadruple-precision floating-point numbers without + * having to deal with long doubles at all. + */ + + predicated_print(context, + [&]() { + std::ostringstream oss; + oss.setf( std::ios::scientific, std::ios::floatfield ); + +#ifdef __x86_64__ + oss.precision( std::numeric_limits::digits10 ); + extFloat80_t val_approx; + f128M_to_extF80M(val.get(), &val_approx); + long double _val; + std::memcpy((char*)&_val, (char*)&val_approx, sizeof(long double)); + oss << _val; +#else + oss.precision( std::numeric_limits::digits10 ); + double val_approx = from_softfloat64( f128M_to_f64(val.get()) ); + oss << val_approx; +#endif + context.console_append( oss.str() ); + }); + } + + void interface::printn(name value) { + predicated_print(context, [&]() { context.console_append(value.to_string()); }); + } + + void interface::printhex(legacy_span data ) { + predicated_print(context, [&]() { context.console_append(fc::to_hex(data.data(), data.size())); }); + } +}}} // ns eosio::chain::webassembly diff --git a/libraries/chain/webassembly/context_free.cpp b/libraries/chain/webassembly/context_free.cpp new file mode 100644 index 00000000000..b0dec1a3b6b --- /dev/null +++ b/libraries/chain/webassembly/context_free.cpp @@ -0,0 +1,8 @@ +#include +#include + +namespace eosio { namespace chain { namespace webassembly { + int32_t interface::get_context_free_data( uint32_t index, legacy_span buffer) const { + return context.get_context_free_data( index, buffer.data(), buffer.size() ); + } +}}} // ns eosio::chain::webassembly diff --git a/libraries/chain/webassembly/crypto.cpp b/libraries/chain/webassembly/crypto.cpp new file mode 100644 index 00000000000..890cc9ad5dc --- /dev/null +++ b/libraries/chain/webassembly/crypto.cpp @@ -0,0 +1,103 @@ +#include +#include +#include +#include + +namespace eosio { namespace chain { namespace webassembly { + + void interface::assert_recover_key( legacy_ptr digest, + legacy_span sig, + legacy_span pub ) const { + fc::crypto::signature s; + fc::crypto::public_key p; + datastream ds( sig.data(), sig.size() ); + datastream pubds ( pub.data(), pub.size() ); + + fc::raw::unpack( ds, s ); + fc::raw::unpack( pubds, p ); + + EOS_ASSERT(static_cast(s.which()) < context.db.get().num_supported_key_types, unactivated_signature_type, + "Unactivated signature type used during assert_recover_key"); + EOS_ASSERT(static_cast(p.which()) < context.db.get().num_supported_key_types, unactivated_key_type, + "Unactivated key type used when creating assert_recover_key"); + + if(context.control.is_producing_block()) + EOS_ASSERT(s.variable_size() <= context.control.configured_subjective_signature_length_limit(), + sig_variable_size_limit_exception, "signature variable length component size greater than subjective maximum"); + + auto check = fc::crypto::public_key( s, *digest, false ); + EOS_ASSERT( check == p, crypto_api_exception, "Error expected key different than recovered key" ); + } + + int32_t interface::recover_key( legacy_ptr digest, + legacy_span sig, + legacy_span pub ) const { + fc::crypto::signature s; + datastream ds( sig.data(), sig.size() ); + fc::raw::unpack(ds, s); + + EOS_ASSERT(static_cast(s.which()) < context.db.get().num_supported_key_types, unactivated_signature_type, + "Unactivated signature type used during recover_key"); + + if(context.control.is_producing_block()) + EOS_ASSERT(s.variable_size() <= context.control.configured_subjective_signature_length_limit(), + sig_variable_size_limit_exception, "signature variable length component size greater than subjective maximum"); + + + auto recovered = fc::crypto::public_key(s, *digest, false); + + // the key types newer than the first 2 may be varible in length + if (static_cast(s.which()) >= config::genesis_num_supported_key_types ) { + EOS_ASSERT(pub.size() >= 33, wasm_execution_error, + "destination buffer must at least be able to hold an ECC public key"); + auto packed_pubkey = fc::raw::pack(recovered); + auto copy_size = std::min(pub.size(), packed_pubkey.size()); + std::memcpy(pub.data(), packed_pubkey.data(), copy_size); + return packed_pubkey.size(); + } else { + // legacy behavior, key types 0 and 1 always pack to 33 bytes. + // this will do one less copy for those keys while maintaining the rules of + // [0..33) dest sizes: assert (asserts in fc::raw::pack) + // [33..inf) dest sizes: return packed size (always 33) + datastream out_ds( pub.data(), pub.size() ); + fc::raw::pack(out_ds, recovered); + return out_ds.tellp(); + } + } + + void interface::assert_sha256(legacy_span data, legacy_ptr hash_val) const { + auto result = context.trx_context.hash_with_checktime( data.data(), data.size() ); + EOS_ASSERT( result == *hash_val, crypto_api_exception, "hash mismatch" ); + } + + void interface::assert_sha1(legacy_span data, legacy_ptr hash_val) const { + auto result = context.trx_context.hash_with_checktime( data.data(), data.size() ); + EOS_ASSERT( result == *hash_val, crypto_api_exception, "hash mismatch" ); + } + + void interface::assert_sha512(legacy_span data, legacy_ptr hash_val) const { + auto result = context.trx_context.hash_with_checktime( data.data(), data.size() ); + EOS_ASSERT( result == *hash_val, crypto_api_exception, "hash mismatch" ); + } + + void interface::assert_ripemd160(legacy_span data, legacy_ptr hash_val) const { + auto result = context.trx_context.hash_with_checktime( data.data(), data.size() ); + EOS_ASSERT( result == *hash_val, crypto_api_exception, "hash mismatch" ); + } + + void interface::sha1(legacy_span data, legacy_ptr hash_val) const { + *hash_val = context.trx_context.hash_with_checktime( data.data(), data.size() ); + } + + void interface::sha256(legacy_span data, legacy_ptr hash_val) const { + *hash_val = context.trx_context.hash_with_checktime( data.data(), data.size() ); + } + + void interface::sha512(legacy_span data, legacy_ptr hash_val) const { + *hash_val = context.trx_context.hash_with_checktime( data.data(), data.size() ); + } + + void interface::ripemd160(legacy_span data, legacy_ptr hash_val) const { + *hash_val = context.trx_context.hash_with_checktime( data.data(), data.size() ); + } +}}} // ns eosio::chain::webassembly diff --git a/libraries/chain/webassembly/database.cpp b/libraries/chain/webassembly/database.cpp new file mode 100644 index 00000000000..45afe0ed3f0 --- /dev/null +++ b/libraries/chain/webassembly/database.cpp @@ -0,0 +1,264 @@ +#include +#include + +namespace eosio { namespace chain { namespace webassembly { + /** + * interface for primary index + */ + int32_t interface::db_store_i64( uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, legacy_span buffer ) { + return context.db_get_context().db_store_i64( scope, table, account_name(payer), id, buffer.data(), buffer.size() ); + } + void interface::db_update_i64( int32_t itr, uint64_t payer, legacy_span buffer ) { + context.db_get_context().db_update_i64( itr, account_name(payer), buffer.data(), buffer.size() ); + } + void interface::db_remove_i64( int32_t itr ) { + context.db_get_context().db_remove_i64( itr ); + } + int32_t interface::db_get_i64( int32_t itr, legacy_span buffer ) { + return context.db_get_context().db_get_i64( itr, buffer.data(), buffer.size() ); + } + int32_t interface::db_next_i64( int32_t itr, legacy_ptr primary ) { + return context.db_get_context().db_next_i64(itr, *primary); + } + int32_t interface::db_previous_i64( int32_t itr, legacy_ptr primary ) { + return context.db_get_context().db_previous_i64(itr, *primary); + } + int32_t interface::db_find_i64( uint64_t code, uint64_t scope, uint64_t table, uint64_t id ) { + return context.db_get_context().db_find_i64( code, scope, table, id ); + } + int32_t interface::db_lowerbound_i64( uint64_t code, uint64_t scope, uint64_t table, uint64_t id ) { + return context.db_get_context().db_lowerbound_i64( code, scope, table, id ); + } + int32_t interface::db_upperbound_i64( uint64_t code, uint64_t scope, uint64_t table, uint64_t id ) { + return context.db_get_context().db_upperbound_i64( code, scope, table, id ); + } + int32_t interface::db_end_i64( uint64_t code, uint64_t scope, uint64_t table ) { + return context.db_get_context().db_end_i64( code, scope, table ); + } + + /** + * interface for uint64_t secondary + */ + int32_t interface::db_idx64_store( uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, legacy_ptr secondary ) { + return context.db_get_context().db_idx64_store( scope, table, account_name(payer), id, *secondary ); + } + void interface::db_idx64_update( int32_t iterator, uint64_t payer, legacy_ptr secondary ) { + context.db_get_context().db_idx64_update( iterator, account_name(payer), *secondary ); + } + void interface::db_idx64_remove( int32_t iterator ) { + context.db_get_context().db_idx64_remove( iterator ); + } + int32_t interface::db_idx64_find_secondary( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { + return context.db_get_context().db_idx64_find_secondary(code, scope, table, *secondary, *primary); + } + int32_t interface::db_idx64_find_primary( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, uint64_t primary ) { + return context.db_get_context().db_idx64_find_primary(code, scope, table, *secondary, primary); + } + int32_t interface::db_idx64_lowerbound( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { + const int32_t ret = context.db_get_context().db_idx64_lowerbound(code, scope, table, *secondary, *primary); + (void)legacy_ptr(std::move(secondary)); + (void)legacy_ptr(std::move(primary)); + return ret; + } + int32_t interface::db_idx64_upperbound( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { + const int32_t ret = context.db_get_context().db_idx64_upperbound(code, scope, table, *secondary, *primary); + (void)legacy_ptr(std::move(secondary)); + (void)legacy_ptr(std::move(primary)); + return ret; + } + int32_t interface::db_idx64_end( uint64_t code, uint64_t scope, uint64_t table ) { + return context.db_get_context().db_idx64_end(code, scope, table); + } + int32_t interface::db_idx64_next( int32_t iterator, legacy_ptr primary ) { + return context.db_get_context().db_idx64_next(iterator, *primary); + } + int32_t interface::db_idx64_previous( int32_t iterator, legacy_ptr primary ) { + return context.db_get_context().db_idx64_previous(iterator, *primary); + } + + + /** + * interface for uint128_t secondary + */ + int32_t interface::db_idx128_store( uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, legacy_ptr secondary ) { + return context.db_get_context().db_idx128_store( scope, table, account_name(payer), id, *secondary ); + } + void interface::db_idx128_update( int32_t iterator, uint64_t payer, legacy_ptr secondary ) { + return context.db_get_context().db_idx128_update( iterator, account_name(payer), *secondary ); + } + void interface::db_idx128_remove( int32_t iterator ) { + return context.db_get_context().db_idx128_remove( iterator ); + } + int32_t interface::db_idx128_find_secondary( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { + return context.db_get_context().db_idx128_find_secondary(code, scope, table, *secondary, *primary); + } + int32_t interface::db_idx128_find_primary( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, uint64_t primary ) { + return context.db_get_context().db_idx128_find_primary(code, scope, table, *secondary, primary); + } + int32_t interface::db_idx128_lowerbound( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { + int32_t result = context.db_get_context().db_idx128_lowerbound(code, scope, table, *secondary, *primary); + (void)legacy_ptr(std::move(secondary)); + (void)legacy_ptr(std::move(primary)); + return result; + } + int32_t interface::db_idx128_upperbound( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { + int32_t result = context.db_get_context().db_idx128_upperbound(code, scope, table, *secondary, *primary); + (void)legacy_ptr(std::move(secondary)); + (void)legacy_ptr(std::move(primary)); + return result; + } + int32_t interface::db_idx128_end( uint64_t code, uint64_t scope, uint64_t table ) { + return context.db_get_context().db_idx128_end(code, scope, table); + } + int32_t interface::db_idx128_next( int32_t iterator, legacy_ptr primary ) { + return context.db_get_context().db_idx128_next(iterator, *primary); + } + int32_t interface::db_idx128_previous( int32_t iterator, legacy_ptr primary ) { + return context.db_get_context().db_idx128_previous(iterator, *primary); + } + + /** + * interface for 256-bit interger secondary + */ + inline static constexpr uint32_t idx256_array_size = 2; + int32_t interface::db_idx256_store( uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, legacy_span data ) { + EOS_ASSERT( data.size() == idx256_array_size, + db_api_exception, + "invalid size of secondary key array for idx256: given ${given} bytes but expected ${expected} bytes", + ("given",data.size())("expected", idx256_array_size) ); + return context.db_get_context().db_idx256_store(scope, table, account_name(payer), id, data.data()); + } + void interface::db_idx256_update( int32_t iterator, uint64_t payer, legacy_span data ) { + EOS_ASSERT( data.size() == idx256_array_size, + db_api_exception, + "invalid size of secondary key array for idx256: given ${given} bytes but expected ${expected} bytes", + ("given",data.size())("expected", idx256_array_size) ); + return context.db_get_context().db_idx256_update(iterator, account_name(payer), data.data()); + } + void interface::db_idx256_remove( int32_t iterator ) { + return context.db_get_context().db_idx256_remove(iterator); + } + int32_t interface::db_idx256_find_secondary( uint64_t code, uint64_t scope, uint64_t table, legacy_span data, legacy_ptr primary ) { + EOS_ASSERT( data.size() == idx256_array_size, + db_api_exception, + "invalid size of secondary key array for idx256: given ${given} bytes but expected ${expected} bytes", + ("given",data.size())("expected", idx256_array_size) ); + return context.db_get_context().db_idx256_find_secondary(code, scope, table, data.data(), *primary); + } + int32_t interface::db_idx256_find_primary( uint64_t code, uint64_t scope, uint64_t table, legacy_span data, uint64_t primary ) { + EOS_ASSERT( data.size() == idx256_array_size, + db_api_exception, + "invalid size of secondary key array for idx256: given ${given} bytes but expected ${expected} bytes", + ("given",data.size())("expected", idx256_array_size) ); + return context.db_get_context().db_idx256_find_primary(code, scope, table, data.data(), primary); + } + int32_t interface::db_idx256_lowerbound( uint64_t code, uint64_t scope, uint64_t table, legacy_span data, legacy_ptr primary ) { + EOS_ASSERT( data.size() == idx256_array_size, + db_api_exception, + "invalid size of secondary key array for idx256: given ${given} bytes but expected ${expected} bytes", + ("given",data.size())("expected", idx256_array_size) ); + int32_t result = context.db_get_context().db_idx256_lowerbound(code, scope, table, data.data(), *primary); + (void)legacy_span(std::move(data)); + (void)legacy_ptr(std::move(primary)); + return result; + } + int32_t interface::db_idx256_upperbound( uint64_t code, uint64_t scope, uint64_t table, legacy_span data, legacy_ptr primary ) { + EOS_ASSERT( data.size() == idx256_array_size, + db_api_exception, + "invalid size of secondary key array for idx256: given ${given} bytes but expected ${expected} bytes", + ("given",data.size())("expected", idx256_array_size) ); + int32_t result = context.db_get_context().db_idx256_upperbound(code, scope, table, data.data(), *primary); + (void)legacy_span(std::move(data)); + (void)legacy_ptr(std::move(primary)); + return result; + } + int32_t interface::db_idx256_end( uint64_t code, uint64_t scope, uint64_t table ) { + return context.db_get_context().db_idx256_end(code, scope, table); + } + int32_t interface::db_idx256_next( int32_t iterator, legacy_ptr primary ) { + return context.db_get_context().db_idx256_next(iterator, *primary); + } + int32_t interface::db_idx256_previous( int32_t iterator, legacy_ptr primary ) { + return context.db_get_context().db_idx256_previous(iterator, *primary); + } + + /** + * interface for double secondary + */ + int32_t interface::db_idx_double_store( uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, legacy_ptr secondary ) { + return context.db_get_context().db_idx_double_store( scope, table, account_name(payer), id, *secondary ); + } + void interface::db_idx_double_update( int32_t iterator, uint64_t payer, legacy_ptr secondary ) { + return context.db_get_context().db_idx_double_update( iterator, account_name(payer), *secondary ); + } + void interface::db_idx_double_remove( int32_t iterator ) { + return context.db_get_context().db_idx_double_remove( iterator ); + } + int32_t interface::db_idx_double_find_secondary( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { + return context.db_get_context().db_idx_double_find_secondary(code, scope, table, *secondary, *primary); + } + int32_t interface::db_idx_double_find_primary( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, uint64_t primary ) { + return context.db_get_context().db_idx_double_find_primary(code, scope, table, *secondary, primary); + } + int32_t interface::db_idx_double_lowerbound( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { + int32_t result = context.db_get_context().db_idx_double_lowerbound(code, scope, table, *secondary, *primary); + (void)legacy_ptr(std::move(secondary)); + (void)legacy_ptr(std::move(primary)); + return result; + } + int32_t interface::db_idx_double_upperbound( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { + int32_t result = context.db_get_context().db_idx_double_upperbound(code, scope, table, *secondary, *primary); + (void)legacy_ptr(std::move(secondary)); + (void)legacy_ptr(std::move(primary)); + return result; + } + int32_t interface::db_idx_double_end( uint64_t code, uint64_t scope, uint64_t table ) { + return context.db_get_context().db_idx_double_end(code, scope, table); + } + int32_t interface::db_idx_double_next( int32_t iterator, legacy_ptr primary ) { + return context.db_get_context().db_idx_double_next(iterator, *primary); + } + int32_t interface::db_idx_double_previous( int32_t iterator, legacy_ptr primary ) { + return context.db_get_context().db_idx_double_previous(iterator, *primary); + } + + /** + * interface for long double secondary + */ + int32_t interface::db_idx_long_double_store( uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, legacy_ptr secondary ) { + return context.db_get_context().db_idx_long_double_store( scope, table, account_name(payer), id, *secondary ); + } + void interface::db_idx_long_double_update( int32_t iterator, uint64_t payer, legacy_ptr secondary ) { + return context.db_get_context().db_idx_long_double_update( iterator, account_name(payer), *secondary ); + } + void interface::db_idx_long_double_remove( int32_t iterator ) { + return context.db_get_context().db_idx_long_double_remove( iterator ); + } + int32_t interface::db_idx_long_double_find_secondary( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { + return context.db_get_context().db_idx_long_double_find_secondary(code, scope, table, *secondary, *primary); + } + int32_t interface::db_idx_long_double_find_primary( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, uint64_t primary ) { + return context.db_get_context().db_idx_long_double_find_primary(code, scope, table, *secondary, primary); + } + int32_t interface::db_idx_long_double_lowerbound( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { + int32_t result = context.db_get_context().db_idx_long_double_lowerbound(code, scope, table, *secondary, *primary); + (void)legacy_ptr(std::move(secondary)); + (void)legacy_ptr(std::move(primary)); + return result; + } + int32_t interface::db_idx_long_double_upperbound( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { + int32_t result = context.db_get_context().db_idx_long_double_upperbound(code, scope, table, *secondary, *primary); + (void)legacy_ptr(std::move(secondary)); + (void)legacy_ptr(std::move(primary)); + return result; + } + int32_t interface::db_idx_long_double_end( uint64_t code, uint64_t scope, uint64_t table ) { + return context.db_get_context().db_idx_long_double_end(code, scope, table); + } + int32_t interface::db_idx_long_double_next( int32_t iterator, legacy_ptr primary ) { + return context.db_get_context().db_idx_long_double_next(iterator, *primary); + } + int32_t interface::db_idx_long_double_previous( int32_t iterator, legacy_ptr primary ) { + return context.db_get_context().db_idx_long_double_previous(iterator, *primary); + } +}}} // ns eosio::chain::webassembly diff --git a/libraries/chain/webassembly/eos-vm.cpp b/libraries/chain/webassembly/eos-vm.cpp deleted file mode 100644 index 99ca29ed72e..00000000000 --- a/libraries/chain/webassembly/eos-vm.cpp +++ /dev/null @@ -1,114 +0,0 @@ -#include -#include -#include -#include -//eos-vm includes -#include - -namespace eosio { namespace chain { namespace webassembly { namespace eos_vm_runtime { - -using namespace eosio::vm; - -namespace wasm_constraints = eosio::chain::wasm_constraints; - -namespace { - - struct checktime_watchdog { - checktime_watchdog(transaction_checktime_timer& timer) : _timer(timer) {} - template - struct guard { - guard(transaction_checktime_timer& timer, F&& func) - : _timer(timer), _func(static_cast(func)) { - _timer.set_expiration_callback(&callback, this); - if(_timer.expired) { - _func(); // it's harmless if _func is invoked twice - } - } - ~guard() { - _timer.set_expiration_callback(nullptr, nullptr); - } - static void callback(void* data) { - guard* self = static_cast(data); - self->_func(); - } - transaction_checktime_timer& _timer; - F _func; - }; - template - guard scoped_run(F&& func) { - return guard{_timer, static_cast(func)}; - } - transaction_checktime_timer& _timer; - }; - -} - -template -class eos_vm_instantiated_module : public wasm_instantiated_module_interface { - using backend_t = backend; - public: - - eos_vm_instantiated_module(eos_vm_runtime* runtime, std::unique_ptr mod) : - _runtime(runtime), - _instantiated_module(std::move(mod)) {} - - void apply(apply_context& context) override { - _instantiated_module->set_wasm_allocator(&context.control.get_wasm_allocator()); - _runtime->_bkend = _instantiated_module.get(); - auto fn = [&]() { - _runtime->_bkend->initialize(&context); - const auto& res = _runtime->_bkend->call( - &context, "env", "apply", context.get_receiver().to_uint64_t(), - context.get_action().account.to_uint64_t(), - context.get_action().name.to_uint64_t()); - }; - try { - checktime_watchdog wd(context.trx_context.transaction_timer); - _runtime->_bkend->timed_run(wd, fn); - } catch(eosio::vm::timeout_exception&) { - context.trx_context.checktime(); - } catch(eosio::vm::wasm_memory_exception& e) { - FC_THROW_EXCEPTION(wasm_execution_error, "access violation"); - } catch(eosio::vm::exception& e) { - // FIXME: Do better translation - FC_THROW_EXCEPTION(wasm_execution_error, "something went wrong..."); - } - _runtime->_bkend = nullptr; - } - - private: - eos_vm_runtime* _runtime; - std::unique_ptr _instantiated_module; -}; - -template -eos_vm_runtime::eos_vm_runtime() {} - -template -void eos_vm_runtime::immediately_exit_currently_running_module() { - throw wasm_exit{}; -} - -template -bool eos_vm_runtime::inject_module(IR::Module& module) { - return false; -} - -template -std::unique_ptr eos_vm_runtime::instantiate_module(const char* code_bytes, size_t code_size, std::vector, - const digest_type&, const uint8_t&, const uint8_t&) { - using backend_t = backend; - try { - wasm_code_ptr code((uint8_t*)code_bytes, code_size); - std::unique_ptr bkend = std::make_unique(code, code_size); - registered_host_functions::resolve(bkend->get_module()); - return std::make_unique>(this, std::move(bkend)); - } catch(eosio::vm::exception& e) { - FC_THROW_EXCEPTION(wasm_execution_error, "Error building eos-vm interp: ${e}", ("e", e.what())); - } -} - -template class eos_vm_runtime; -template class eos_vm_runtime; - -}}}} diff --git a/libraries/chain/webassembly/kv_database.cpp b/libraries/chain/webassembly/kv_database.cpp new file mode 100644 index 00000000000..cc0bb30ae0a --- /dev/null +++ b/libraries/chain/webassembly/kv_database.cpp @@ -0,0 +1,64 @@ +#include +#include + +namespace eosio { namespace chain { namespace webassembly { + int64_t interface::kv_erase(uint64_t contract, span key) { + return context.kv_erase(contract, key.data(), key.size()); + } + + int64_t interface::kv_set(uint64_t contract, span key, span value, account_name payer) { + return context.kv_set(contract, key.data(), key.size(), value.data(), value.size(), payer); + } + + bool interface::kv_get(uint64_t contract, span key, uint32_t* value_size) { + return context.kv_get(contract, key.data(), key.size(), *value_size); + } + + uint32_t interface::kv_get_data(uint32_t offset, span data) { + return context.kv_get_data(offset, data.data(), data.size()); + } + + uint32_t interface::kv_it_create(uint64_t contract, span prefix) { + return context.kv_it_create(contract, prefix.data(), prefix.size()); + } + + void interface::kv_it_destroy(uint32_t itr) { + return context.kv_it_destroy(itr); + } + + int32_t interface::kv_it_status(uint32_t itr) { + return context.kv_it_status(itr); + } + + int32_t interface::kv_it_compare(uint32_t itr_a, uint32_t itr_b) { + return context.kv_it_compare(itr_a, itr_b); + } + + int32_t interface::kv_it_key_compare(uint32_t itr, span key) { + return context.kv_it_key_compare(itr, key.data(), key.size()); + } + + int32_t interface::kv_it_move_to_end(uint32_t itr) { + return context.kv_it_move_to_end(itr); + } + + int32_t interface::kv_it_next(uint32_t itr, uint32_t* found_key_size, uint32_t* found_value_size) { + return context.kv_it_next(itr, found_key_size, found_value_size); + } + + int32_t interface::kv_it_prev(uint32_t itr, uint32_t* found_key_size, uint32_t* found_value_size) { + return context.kv_it_prev(itr, found_key_size, found_value_size); + } + + int32_t interface::kv_it_lower_bound(uint32_t itr, span key, uint32_t* found_key_size, uint32_t* found_value_size) { + return context.kv_it_lower_bound(itr, key.data(), key.size(), found_key_size, found_value_size); + } + + int32_t interface::kv_it_key(uint32_t itr, uint32_t offset, span dest, uint32_t* actual_size) { + return context.kv_it_key(itr, offset, dest.data(), dest.size(), *actual_size); + } + + int32_t interface::kv_it_value(uint32_t itr, uint32_t offset, span dest, uint32_t* actual_size) { + return context.kv_it_value(itr, offset, dest.data(), dest.size(), *actual_size); + } +}}} // ns eosio::chain::webassembly diff --git a/libraries/chain/webassembly/memory.cpp b/libraries/chain/webassembly/memory.cpp new file mode 100644 index 00000000000..39b1c9e26ab --- /dev/null +++ b/libraries/chain/webassembly/memory.cpp @@ -0,0 +1,27 @@ +#include + +namespace eosio { namespace chain { namespace webassembly { + void* interface::memcpy( memcpy_params args ) const { + auto [dest, src, length] = args; + EOS_ASSERT((size_t)(std::abs((ptrdiff_t)(char*)dest - (ptrdiff_t)(const char*)src)) >= length, + overlapping_memory_error, "memcpy can only accept non-aliasing pointers"); + return (char *)std::memcpy((char*)dest, (const char*)src, length); + } + + void* interface::memmove( memcpy_params args ) const { + auto [dest, src, length] = args; + return (char *)std::memmove((char*)dest, (const char*)src, length); + } + + int32_t interface::memcmp( memcmp_params args ) const { + auto [dest, src, length] = args; + int32_t ret = std::memcmp((const char*)dest, (const char*)src, length); + return ret < 0 ? -1 : ret > 0 ? 1 : 0; + } + + void* interface::memset( memset_params args ) const { + auto [dest, value, length] = args; + return (char *)std::memset( (char*)dest, value, length ); + } + +}}} // ns eosio::chain::webassembly diff --git a/libraries/chain/webassembly/permission.cpp b/libraries/chain/webassembly/permission.cpp new file mode 100644 index 00000000000..c5b521b8513 --- /dev/null +++ b/libraries/chain/webassembly/permission.cpp @@ -0,0 +1,89 @@ +#include +#include +#include +#include + +namespace eosio { namespace chain { namespace webassembly { + void unpack_provided_keys( flat_set& keys, const char* pubkeys_data, uint32_t pubkeys_size ) { + keys.clear(); + if( pubkeys_size == 0 ) return; + + keys = fc::raw::unpack>( pubkeys_data, pubkeys_size ); + } + + void unpack_provided_permissions( flat_set& permissions, const char* perms_data, uint32_t perms_size ) { + permissions.clear(); + if( perms_size == 0 ) return; + + permissions = fc::raw::unpack>( perms_data, perms_size ); + } + + bool interface::check_transaction_authorization( legacy_span trx_data, + legacy_span pubkeys_data, + legacy_span perms_data ) const { + transaction trx = fc::raw::unpack( trx_data.data(), trx_data.size() ); + + flat_set provided_keys; + unpack_provided_keys( provided_keys, pubkeys_data.data(), pubkeys_data.size() ); + + flat_set provided_permissions; + unpack_provided_permissions( provided_permissions, perms_data.data(), perms_data.size() ); + + try { + context.control + .get_authorization_manager() + .check_authorization( trx.actions, + provided_keys, + provided_permissions, + fc::seconds(trx.delay_sec), + std::bind(&transaction_context::checktime, &context.trx_context), + false + ); + return true; + } catch( const authorization_exception& e ) {} + + return false; + } + + bool interface::check_permission_authorization( account_name account, permission_name permission, + legacy_span pubkeys_data, + legacy_span perms_data, + uint64_t delay_us ) const { + EOS_ASSERT( delay_us <= static_cast(std::numeric_limits::max()), + action_validate_exception, "provided delay is too large" ); + + flat_set provided_keys; + unpack_provided_keys( provided_keys, pubkeys_data.data(), pubkeys_data.size() ); + + flat_set provided_permissions; + unpack_provided_permissions( provided_permissions, perms_data.data(), perms_data.size() ); + + try { + context.control + .get_authorization_manager() + .check_authorization( account, + permission, + provided_keys, + provided_permissions, + fc::microseconds(delay_us), + std::bind(&transaction_context::checktime, &context.trx_context), + false + ); + return true; + } catch( const authorization_exception& e ) {} + + return false; + } + + int64_t interface::get_permission_last_used( account_name account, permission_name permission ) const { + const auto& am = context.control.get_authorization_manager(); + return am.get_permission_last_used( am.get_permission({account, permission}) ).time_since_epoch().count(); + }; + + int64_t interface::get_account_creation_time( account_name account ) const { + const auto* acct = context.db.find(account); + EOS_ASSERT( acct != nullptr, action_validate_exception, + "account '${account}' does not exist", ("account", account) ); + return time_point(acct->creation_date).time_since_epoch().count(); + } +}}} // ns eosio::chain::webassembly diff --git a/libraries/chain/webassembly/privileged.cpp b/libraries/chain/webassembly/privileged.cpp new file mode 100644 index 00000000000..abde73a9304 --- /dev/null +++ b/libraries/chain/webassembly/privileged.cpp @@ -0,0 +1,290 @@ +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace eosio { namespace chain { namespace webassembly { + + int interface::is_feature_active( int64_t feature_name ) const { return false; } + + void interface::activate_feature( int64_t feature_name ) const { + EOS_ASSERT( false, unsupported_feature, "Unsupported Hardfork Detected" ); + } + + void interface::preactivate_feature( legacy_ptr feature_digest ) { + context.control.preactivate_feature( context.get_action_id(), *feature_digest ); + } + + /** + * Deprecated in favor of set_resource_limit. + */ + void interface::set_resource_limits( account_name account, int64_t ram_bytes, int64_t net_weight, int64_t cpu_weight ) { + EOS_ASSERT(ram_bytes >= -1, wasm_execution_error, "invalid value for ram resource limit expected [-1,INT64_MAX]"); + EOS_ASSERT(net_weight >= -1, wasm_execution_error, "invalid value for net resource weight expected [-1,INT64_MAX]"); + EOS_ASSERT(cpu_weight >= -1, wasm_execution_error, "invalid value for cpu resource weight expected [-1,INT64_MAX]"); + if( context.control.get_mutable_resource_limits_manager().set_account_limits(account, ram_bytes, net_weight, cpu_weight) ) { + context.trx_context.validate_ram_usage.insert( account ); + } + } + + /** + * Deprecated in favor of get_resource_limit. + */ + void interface::get_resource_limits( account_name account, legacy_ptr ram_bytes, legacy_ptr net_weight, legacy_ptr cpu_weight ) const { + context.control.get_resource_limits_manager().get_account_limits( account, *ram_bytes, *net_weight, *cpu_weight); + (void)legacy_ptr(std::move(ram_bytes)); + (void)legacy_ptr(std::move(net_weight)); + (void)legacy_ptr(std::move(cpu_weight)); + } + + void interface::set_resource_limit( account_name account, name resource, int64_t limit ) { + EOS_ASSERT(limit >= -1, wasm_execution_error, "invalid value for ${resource} resource limit expected [-1,INT64_MAX]", ("resource", resource)); + auto& manager = context.control.get_mutable_resource_limits_manager(); + if( resource == string_to_name("ram") ) { + int64_t ram, net, cpu; + manager.get_account_limits(account, ram, net, cpu); + if( manager.set_account_limits( account, limit, net, cpu ) ) { + context.trx_context.validate_ram_usage.insert( account ); + } + } else if( resource == string_to_name("net") ) { + int64_t ram, net, cpu; + manager.get_account_limits(account, ram, net, cpu); + manager.set_account_limits( account, ram, limit, cpu ); + } else if( resource == string_to_name("cpu") ) { + int64_t ram, net, cpu; + manager.get_account_limits(account, ram, net, cpu); + manager.set_account_limits( account, ram, net, limit ); + } else { + EOS_THROW(wasm_execution_error, "unknown resource ${resource}", ("resource", resource)); + } + } + + int64_t interface::get_resource_limit( account_name account, name resource ) const { + const auto& manager = context.control.get_resource_limits_manager(); + if( resource == string_to_name("ram") ) { + int64_t ram, net, cpu; + manager.get_account_limits( account, ram, net, cpu ); + return ram; + } else if( resource == string_to_name("net") ) { + int64_t ram, net, cpu; + manager.get_account_limits( account, ram, net, cpu ); + return net; + } else if( resource == string_to_name("cpu") ) { + int64_t ram, net, cpu; + manager.get_account_limits( account, ram, net, cpu ); + return cpu; + } else { + EOS_THROW(wasm_execution_error, "unknown resource ${resource}", ("resource", resource)); + } + } + + int64_t set_proposed_producers_common( apply_context& context, vector && producers, bool validate_keys ) { + EOS_ASSERT(producers.size() <= config::max_producers, wasm_execution_error, "Producer schedule exceeds the maximum producer count for this chain"); + EOS_ASSERT( producers.size() > 0 + || !context.control.is_builtin_activated( builtin_protocol_feature_t::disallow_empty_producer_schedule ), + wasm_execution_error, + "Producer schedule cannot be empty" + ); + + const int64_t num_supported_key_types = context.db.get().num_supported_key_types; + + // check that producers are unique + std::set unique_producers; + for (const auto& p: producers) { + EOS_ASSERT( context.is_account(p.producer_name), wasm_execution_error, "producer schedule includes a nonexisting account" ); + std::visit([&p, num_supported_key_types, validate_keys](const auto& a) { + uint32_t sum_weights = 0; + std::set unique_keys; + for (const auto& kw: a.keys ) { + EOS_ASSERT( kw.key.which() < num_supported_key_types, unactivated_key_type, + "Unactivated key type used in proposed producer schedule"); + + if( validate_keys ) { + EOS_ASSERT( kw.key.valid(), wasm_execution_error, "producer schedule includes an invalid key" ); + } + + if (std::numeric_limits::max() - sum_weights <= kw.weight) { + sum_weights = std::numeric_limits::max(); + } else { + sum_weights += kw.weight; + } + + unique_keys.insert(kw.key); + } + + EOS_ASSERT( a.keys.size() == unique_keys.size(), wasm_execution_error, "producer schedule includes a duplicated key for ${account}", ("account", p.producer_name)); + EOS_ASSERT( a.threshold > 0, wasm_execution_error, "producer schedule includes an authority with a threshold of 0 for ${account}", ("account", p.producer_name)); + EOS_ASSERT( sum_weights >= a.threshold, wasm_execution_error, "producer schedule includes an unsatisfiable authority for ${account}", ("account", p.producer_name)); + }, p.authority); + + unique_producers.insert(p.producer_name); + } + EOS_ASSERT( producers.size() == unique_producers.size(), wasm_execution_error, "duplicate producer name in producer schedule" ); + + return context.control.set_proposed_producers( std::move(producers) ); + } + + uint32_t interface::get_wasm_parameters_packed( span packed_parameters, uint32_t max_version ) const { + auto& gpo = context.control.get_global_properties(); + auto& params = gpo.wasm_configuration; + uint32_t version = std::min( max_version, uint32_t(0) ); + + auto s = fc::raw::pack_size( params ); + if ( packed_parameters.size() == 0 ) + return s; + + if ( s <= packed_parameters.size() ) { + datastream ds( packed_parameters.data(), s ); + fc::raw::pack(ds, version); + fc::raw::pack(ds, params); + } + return s; + } + void interface::set_wasm_parameters_packed( span packed_parameters ) { + datastream ds( packed_parameters.data(), packed_parameters.size() ); + uint32_t version; + chain::wasm_config cfg; + fc::raw::unpack(ds, version); + EOS_ASSERT(version == 0, wasm_config_unknown_version, "set_wasm_parameters_packed: Unknown version: ${version}", ("version", version)); + fc::raw::unpack(ds, cfg); + cfg.validate(); + context.db.modify( context.control.get_global_properties(), + [&]( auto& gprops ) { + gprops.wasm_configuration = cfg; + } + ); + } + int64_t interface::set_proposed_producers( legacy_span packed_producer_schedule) { + datastream ds( packed_producer_schedule.data(), packed_producer_schedule.size() ); + std::vector producers; + std::vector old_version; + fc::raw::unpack(ds, old_version); + + /* + * Up-convert the producers + */ + for ( const auto& p : old_version ) { + producers.emplace_back( producer_authority{ p.producer_name, block_signing_authority_v0{ 1, {{p.block_signing_key, 1}} } } ); + } + + return set_proposed_producers_common( context, std::move(producers), true ); + } + + int64_t interface::set_proposed_producers_ex( uint64_t packed_producer_format, legacy_span packed_producer_schedule) { + if (packed_producer_format == 0) { + return set_proposed_producers(std::move(packed_producer_schedule)); + } else if (packed_producer_format == 1) { + datastream ds( packed_producer_schedule.data(), packed_producer_schedule.size() ); + vector producers; + + fc::raw::unpack(ds, producers); + return set_proposed_producers_common( context, std::move(producers), false); + } else { + EOS_THROW(wasm_execution_error, "Producer schedule is in an unknown format!"); + } + } + + uint32_t interface::get_blockchain_parameters_packed( legacy_span packed_blockchain_parameters ) const { + auto& gpo = context.control.get_global_properties(); + + auto s = fc::raw::pack_size( gpo.configuration.v0() ); + if( packed_blockchain_parameters.size() == 0 ) return s; + + if ( s <= packed_blockchain_parameters.size() ) { + datastream ds( packed_blockchain_parameters.data(), s ); + fc::raw::pack(ds, gpo.configuration.v0()); + return s; + } + return 0; + } + + void interface::set_blockchain_parameters_packed( legacy_span packed_blockchain_parameters ) { + datastream ds( packed_blockchain_parameters.data(), packed_blockchain_parameters.size() ); + chain::chain_config_v0 cfg; + fc::raw::unpack(ds, cfg); + cfg.validate(); + context.db.modify( context.control.get_global_properties(), + [&]( auto& gprops ) { + gprops.configuration = cfg; + }); + } + + uint32_t interface::get_parameters_packed( span packed_parameter_ids, span packed_parameters) const{ + datastream ds_ids( packed_parameter_ids.data(), packed_parameter_ids.size() ); + + chain::chain_config cfg = context.control.get_global_properties().configuration; + std::vector ids; + fc::raw::unpack(ds_ids, ids); + const config_range config_range(cfg, std::move(ids), {context.control}); + + auto size = fc::raw::pack_size( config_range ); + if( packed_parameters.size() == 0 ) return size; + + EOS_ASSERT(size <= packed_parameters.size(), + chain::config_parse_error, + "get_parameters_packed: buffer size is smaller than ${size}", ("size", size)); + + datastream ds( packed_parameters.data(), size ); + fc::raw::pack( ds, config_range ); + return size; + } + + void interface::set_parameters_packed( span packed_parameters ){ + datastream ds( packed_parameters.data(), packed_parameters.size() ); + + chain::chain_config cfg = context.control.get_global_properties().configuration; + config_range config_range(cfg, {context.control}); + + fc::raw::unpack(ds, config_range); + + config_range.config.validate(); + context.db.modify( context.control.get_global_properties(), + [&]( auto& gprops ) { + gprops.configuration = config_range.config; + }); + } + + uint32_t interface::get_kv_parameters_packed( span packed_kv_parameters, uint32_t max_version ) const { + const auto& gpo = context.control.get_global_properties(); + const auto& params = gpo.kv_configuration; + uint32_t version = std::min( max_version, uint32_t(0) ); + + auto s = fc::raw::pack_size( version ) + fc::raw::pack_size( params ); + + if ( s <= packed_kv_parameters.size() ) { + datastream ds( packed_kv_parameters.data(), s ); + fc::raw::pack(ds, version); + fc::raw::pack(ds, params); + } + return s; + } + + void interface::set_kv_parameters_packed( span packed_kv_parameters ) { + datastream ds( packed_kv_parameters.data(), packed_kv_parameters.size() ); + uint32_t version; + chain::kv_database_config cfg; + fc::raw::unpack(ds, version); + EOS_ASSERT(version == 0, kv_unknown_parameters_version, "set_kv_parameters_packed: Unknown version: ${version}", ("version", version)); + fc::raw::unpack(ds, cfg); + context.db.modify( context.control.get_global_properties(), + [&]( auto& gprops ) { + gprops.kv_configuration = cfg; + }); + } + + bool interface::is_privileged( account_name n ) const { + return context.db.get( n ).is_privileged(); + } + + void interface::set_privileged( account_name n, bool is_priv ) { + const auto& a = context.db.get( n ); + context.db.modify( a, [&]( auto& ma ){ + ma.set_privileged( is_priv ); + }); + } +}}} // ns eosio::chain::webassembly diff --git a/libraries/chain/webassembly/producer.cpp b/libraries/chain/webassembly/producer.cpp new file mode 100644 index 00000000000..6a1ab14948a --- /dev/null +++ b/libraries/chain/webassembly/producer.cpp @@ -0,0 +1,17 @@ +#include +#include + +namespace eosio { namespace chain { namespace webassembly { + int32_t interface::get_active_producers( legacy_span producers ) const { + auto active_producers = context.get_active_producers(); + + size_t len = active_producers.size(); + auto s = len * sizeof(chain::account_name); + if( producers.size_bytes() == 0 ) return s; + + auto copy_size = std::min( producers.size(), s ); + std::memcpy( producers.data(), active_producers.data(), copy_size ); + + return copy_size; + } +}}} // ns eosio::chain::webassembly diff --git a/libraries/chain/webassembly/eos-vm-oc.cpp b/libraries/chain/webassembly/runtimes/eos-vm-oc.cpp similarity index 87% rename from libraries/chain/webassembly/eos-vm-oc.cpp rename to libraries/chain/webassembly/runtimes/eos-vm-oc.cpp index 3db4399f133..c472321cd78 100644 --- a/libraries/chain/webassembly/eos-vm-oc.cpp +++ b/libraries/chain/webassembly/runtimes/eos-vm-oc.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -36,14 +37,14 @@ class eosvmoc_instantiated_module : public wasm_instantiated_module_interface { }; eosvmoc_runtime::eosvmoc_runtime(const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db) - : cc(data_dir, eosvmoc_config, db), exec(cc) { + : cc(data_dir, eosvmoc_config, db), exec(cc), mem(wasm_constraints::maximum_linear_memory/wasm_constraints::wasm_page_size) { } eosvmoc_runtime::~eosvmoc_runtime() { } std::unique_ptr eosvmoc_runtime::instantiate_module(const char* code_bytes, size_t code_size, std::vector initial_memory, - const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) { + const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) { return std::make_unique(code_hash, vm_type, *this); } diff --git a/libraries/chain/webassembly/eos-vm-oc/About WAVM b/libraries/chain/webassembly/runtimes/eos-vm-oc/About WAVM similarity index 100% rename from libraries/chain/webassembly/eos-vm-oc/About WAVM rename to libraries/chain/webassembly/runtimes/eos-vm-oc/About WAVM diff --git a/libraries/chain/webassembly/eos-vm-oc/LLVMEmitIR.cpp b/libraries/chain/webassembly/runtimes/eos-vm-oc/LLVMEmitIR.cpp similarity index 96% rename from libraries/chain/webassembly/eos-vm-oc/LLVMEmitIR.cpp rename to libraries/chain/webassembly/runtimes/eos-vm-oc/LLVMEmitIR.cpp index 0f4d953f30f..12dbd623362 100644 --- a/libraries/chain/webassembly/eos-vm-oc/LLVMEmitIR.cpp +++ b/libraries/chain/webassembly/runtimes/eos-vm-oc/LLVMEmitIR.cpp @@ -59,6 +59,10 @@ namespace LLVMJIT return "wasmFunc" + std::to_string(functionDefIndex); } + const char* getTableSymbolName() { + return "wasmTable"; + } + bool getFunctionIndexFromExternalName(const char* externalName,Uptr& outFunctionDefIndex) { const char wasmFuncPrefix[] = "wasmFunc"; @@ -130,7 +134,7 @@ namespace LLVMJIT std::vector functionDefs; std::vector importedFunctionOffsets; std::vector globals; - llvm::Constant* defaultTablePointer; + llvm::GlobalVariable* defaultTablePointer; llvm::Constant* defaultTableMaxElementIndex; llvm::Constant* defaultMemoryBase; llvm::Constant* depthCounter; @@ -746,7 +750,8 @@ namespace LLVMJIT "eosvmoc_internal.indirect_call_oob",FunctionType::get(),{}); // Load the type for this table entry. - auto functionTypePointerPointer = CreateInBoundsGEPWAR(irBuilder, moduleContext.defaultTablePointer, functionIndexZExt, emitLiteral((U32)0)); + auto tablePointer = CreateInBoundsGEPWAR(irBuilder, moduleContext.defaultTablePointer, emitLiteral(0), emitLiteral(0)); + auto functionTypePointerPointer = CreateInBoundsGEPWAR(irBuilder, tablePointer, functionIndexZExt, emitLiteral((U32)0)); auto functionTypePointer = irBuilder.CreateLoad(functionTypePointerPointer); auto llvmCalleeType = emitLiteralPointer(calleeType,llvmI8PtrType); @@ -760,7 +765,7 @@ namespace LLVMJIT //If the WASM only contains table elements to function definitions internal to the wasm, we can take a // simple and approach if(moduleContext.tableOnlyHasDefinedFuncs) { - auto functionPointerPointer = CreateInBoundsGEPWAR(irBuilder, moduleContext.defaultTablePointer, functionIndexZExt, emitLiteral((U32)1)); + auto functionPointerPointer = CreateInBoundsGEPWAR(irBuilder, tablePointer, functionIndexZExt, emitLiteral((U32)1)); auto functionInfo = irBuilder.CreateLoad(functionPointerPointer); //offset of code llvm::Value* running_code_start = irBuilder.CreateLoad(emitLiteralPointer((void*)OFFSET_OF_CONTROL_BLOCK_MEMBER(running_code_base), llvmI64Type->getPointerTo(256))); llvm::Value* offset_from_start = irBuilder.CreateAdd(running_code_start, functionInfo); @@ -771,7 +776,7 @@ namespace LLVMJIT if(calleeType->ret != ResultType::none) { push(result); } } else { - auto functionPointerPointer = CreateInBoundsGEPWAR(irBuilder, moduleContext.defaultTablePointer, functionIndexZExt, emitLiteral((U32)1)); + auto functionPointerPointer = CreateInBoundsGEPWAR(irBuilder, tablePointer, functionIndexZExt, emitLiteral((U32)1)); auto functionInfo = irBuilder.CreateLoad(functionPointerPointer); //offset of code auto is_intrnsic = irBuilder.CreateICmpSLT(functionInfo, typedZeroConstants[(Uptr)ValueType::i64]); @@ -824,18 +829,28 @@ namespace LLVMJIT auto value = irBuilder.CreateBitCast(pop(),localPointers[imm.variableIndex]->getType()->getPointerElementType()); irBuilder.CreateStore(value,localPointers[imm.variableIndex]); } + llvm::Value* get_mutable_global_ptr(llvm::Value* global) { + if(global->getType()->isStructTy()) { + llvm::Value* globalsBasePtr = irBuilder.CreateExtractValue(global, 0); + return CreateInBoundsGEPWAR(irBuilder, irBuilder.CreateLoad(globalsBasePtr), irBuilder.CreateExtractValue(global, 1)); + } else if(global->getType()->isPointerTy()) { + return global; + } else { + return nullptr; + } + } void tee_local(GetOrSetVariableImm imm) { WAVM_ASSERT_THROW(imm.variableIndex < localPointers.size()); auto value = irBuilder.CreateBitCast(getTopValue(),localPointers[imm.variableIndex]->getType()->getPointerElementType()); - irBuilder.CreateStore(value,localPointers[imm.variableIndex]); + irBuilder.CreateStore(value,get_mutable_global_ptr(localPointers[imm.variableIndex])); } void get_global(GetOrSetVariableImm imm) { WAVM_ASSERT_THROW(imm.variableIndex < moduleContext.globals.size()); - if(moduleContext.globals[imm.variableIndex]->getType()->isPointerTy()) - push(irBuilder.CreateLoad(moduleContext.globals[imm.variableIndex])); + if(auto* p = get_mutable_global_ptr(moduleContext.globals[imm.variableIndex])) + push(irBuilder.CreateLoad(p)); else push(moduleContext.globals[imm.variableIndex]); } @@ -843,7 +858,7 @@ namespace LLVMJIT { WAVM_ASSERT_THROW(imm.variableIndex < moduleContext.globals.size()); auto value = irBuilder.CreateBitCast(pop(),moduleContext.globals[imm.variableIndex]->getType()->getPointerElementType()); - irBuilder.CreateStore(value,moduleContext.globals[imm.variableIndex]); + irBuilder.CreateStore(value,get_mutable_global_ptr(moduleContext.globals[imm.variableIndex])); } // @@ -1253,11 +1268,19 @@ namespace LLVMJIT importedFunctionOffsets.push_back(ie.ordinal); } - int current_prologue = -8; + intptr_t current_prologue = -8; for(const GlobalDef& global : module.globals.defs) { if(global.type.isMutable) { - globals.push_back(emitLiteralPointer((void*)current_prologue,asLLVMType(global.type.valueType)->getPointerTo(256))); + if(current_prologue >= -(int)memory::max_prologue_size) { + globals.push_back(emitLiteralPointer((void*)current_prologue,asLLVMType(global.type.valueType)->getPointerTo(256))); + } else { + auto baseType = asLLVMType(global.type.valueType)->getPointerTo()->getPointerTo(256); + auto basePtr = emitLiteralPointer((void*)OFFSET_OF_CONTROL_BLOCK_MEMBER(globals), baseType); + auto structTy = llvm::StructType::get(context, {baseType, llvmI64Type}); + I64 typeSize = IR::getTypeBitWidth(global.type.valueType)/8; + globals.push_back(llvm::ConstantStruct::get(structTy, {basePtr, emitLiteral((I64)current_prologue/typeSize)})); + } current_prologue -= 8; } else { @@ -1272,12 +1295,12 @@ namespace LLVMJIT } if(module.tables.size()) { - current_prologue -= 8; //now pointing to LAST element - current_prologue -= 16*(module.tables.defs[0].type.size.min-1); //now pointing to FIRST element auto tableElementType = llvm::StructType::get(context,{llvmI8PtrType, llvmI64Type}); - defaultTablePointer = emitLiteralPointer((void*)current_prologue,tableElementType->getPointerTo(256)); + llvm::Type* tableArrayTy = llvm::ArrayType::get(tableElementType, module.tables.defs[0].type.size.min); + defaultTablePointer = new llvm::GlobalVariable(*llvmModule, tableArrayTy, true, llvm::GlobalValue::ExternalLinkage, llvm::ConstantAggregateZero::get(tableArrayTy), getTableSymbolName()); + defaultTablePointer->setVisibility(llvm::GlobalValue::ProtectedVisibility); // Don't use the GOT. + defaultTablePointer->setExternallyInitialized(true); defaultTableMaxElementIndex = emitLiteral((U64)module.tables.defs[0].type.size.min); - for(const TableSegment& table_segment : module.tableSegments) for(Uptr i = 0; i < table_segment.indices.size(); ++i) if(table_segment.indices[i] < module.functions.imports.size()) @@ -1331,4 +1354,4 @@ namespace LLVMJIT return EmitModuleContext(module).emit(); } } -}}} \ No newline at end of file +}}} diff --git a/libraries/chain/webassembly/eos-vm-oc/LLVMJIT.cpp b/libraries/chain/webassembly/runtimes/eos-vm-oc/LLVMJIT.cpp similarity index 95% rename from libraries/chain/webassembly/eos-vm-oc/LLVMJIT.cpp rename to libraries/chain/webassembly/runtimes/eos-vm-oc/LLVMJIT.cpp index 8345beeb799..8b76b715af5 100644 --- a/libraries/chain/webassembly/eos-vm-oc/LLVMJIT.cpp +++ b/libraries/chain/webassembly/runtimes/eos-vm-oc/LLVMJIT.cpp @@ -193,6 +193,12 @@ namespace LLVMJIT #if PRINT_DISASSEMBLY disassembleFunction((U8*)loadedAddress, symbolSizePair.second); #endif + } else if(symbol.getType() && symbol.getType().get() == llvm::object::SymbolRef::ST_Data && name && *name == getTableSymbolName()) { + Uptr loadedAddress = Uptr(*address); + auto symbolSection = symbol.getSection(); + if(symbolSection) + loadedAddress += (Uptr)o.getSectionLoadAddress(*symbolSection.get()); + table_offset = loadedAddress-(uintptr_t)unitmemorymanager->code->data(); } } } @@ -207,6 +213,7 @@ namespace LLVMJIT std::map function_to_offsets; std::vector final_pic_code; + uintptr_t table_offset = 0; ~JITModule() { @@ -299,7 +306,7 @@ namespace LLVMJIT unsigned num_functions_stack_size_found = 0; for(const auto& stacksizes : jitModule->unitmemorymanager->stack_sizes) { - fc::datastream ds(stacksizes.data(), stacksizes.size()); + fc::datastream ds(reinterpret_cast(stacksizes.data()), stacksizes.size()); while(ds.remaining()) { uint64_t funcaddr; fc::unsigned_int stack_size; @@ -319,7 +326,8 @@ namespace LLVMJIT instantiated_code ret; ret.code = jitModule->final_pic_code; ret.function_offsets = jitModule->function_to_offsets; + ret.table_offset = jitModule->table_offset; return ret; } } -}}} \ No newline at end of file +}}} diff --git a/libraries/chain/webassembly/eos-vm-oc/LLVMJIT.h b/libraries/chain/webassembly/runtimes/eos-vm-oc/LLVMJIT.h similarity index 89% rename from libraries/chain/webassembly/eos-vm-oc/LLVMJIT.h rename to libraries/chain/webassembly/runtimes/eos-vm-oc/LLVMJIT.h index 5b1152f56d6..4d5a685c29d 100644 --- a/libraries/chain/webassembly/eos-vm-oc/LLVMJIT.h +++ b/libraries/chain/webassembly/runtimes/eos-vm-oc/LLVMJIT.h @@ -15,11 +15,13 @@ namespace eosio { namespace chain { namespace eosvmoc { struct instantiated_code { std::vector code; std::map function_offsets; + uintptr_t table_offset; }; namespace LLVMJIT { bool getFunctionIndexFromExternalName(const char* externalName,Uptr& outFunctionDefIndex); + const char* getTableSymbolName(); llvm::Module* emitModule(const IR::Module& module); instantiated_code instantiateModule(const IR::Module& module); } -}}} \ No newline at end of file +}}} diff --git a/libraries/chain/webassembly/eos-vm-oc/code_cache.cpp b/libraries/chain/webassembly/runtimes/eos-vm-oc/code_cache.cpp similarity index 95% rename from libraries/chain/webassembly/eos-vm-oc/code_cache.cpp rename to libraries/chain/webassembly/runtimes/eos-vm-oc/code_cache.cpp index 9d93b8a2111..85403cb5907 100644 --- a/libraries/chain/webassembly/eos-vm-oc/code_cache.cpp +++ b/libraries/chain/webassembly/runtimes/eos-vm-oc/code_cache.cpp @@ -69,12 +69,12 @@ void code_cache_async::wait_on_compile_monitor_message() { } auto [success, message, fds] = read_message_with_fds(_compile_monitor_read_socket); - if(!success || !message.contains()) { + if(!success || !std::holds_alternative(message)) { _ctx.stop(); return; } - _result_queue.push(message.get()); + _result_queue.push(std::get(message)); wait_on_compile_monitor_message(); }); @@ -86,7 +86,7 @@ std::tuple code_cache_async::consume_compile_thread_queue() { size_t bytes_remaining = 0; size_t gotsome = _result_queue.consume_all([&](const wasm_compilation_result_message& result) { if(_outstanding_compiles_and_poison[result.code] == false) { - result.result.visit(overloaded { + std::visit(overloaded { [&](const code_descriptor& cd) { _cache_index.push_front(cd); }, @@ -97,7 +97,7 @@ std::tuple code_cache_async::consume_compile_thread_queue() { [&](const compilation_result_toofull&) { run_eviction_round(); } - }); + }, result.result); } _outstanding_compiles_and_poison.erase(result.code); bytes_remaining = result.cache_free_bytes; @@ -192,14 +192,14 @@ const code_descriptor* const code_cache_sync::get_descriptor_for_code_sync(const write_message_with_fds(_compile_monitor_write_socket, compile_wasm_message{ {code_id, vm_version} }, fds_to_pass); auto [success, message, fds] = read_message_with_fds(_compile_monitor_read_socket); EOS_ASSERT(success, wasm_execution_error, "failed to read response from monitor process"); - EOS_ASSERT(message.contains(), wasm_execution_error, "unexpected response from monitor process"); + EOS_ASSERT(std::holds_alternative(message), wasm_execution_error, "unexpected response from monitor process"); - wasm_compilation_result_message result = message.get(); - EOS_ASSERT(result.result.contains(), wasm_execution_error, "failed to compile wasm"); + wasm_compilation_result_message result = std::get(message); + EOS_ASSERT(std::holds_alternative(result.result), wasm_execution_error, "failed to compile wasm"); check_eviction_threshold(result.cache_free_bytes); - return &*_cache_index.push_front(std::move(result.result.get())).first; + return &*_cache_index.push_front(std::move(std::get(result.result))).first; } code_cache_base::code_cache_base(const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db) : @@ -382,4 +382,4 @@ void code_cache_base::check_eviction_threshold(size_t free_bytes) { run_eviction_round(); } -}}} \ No newline at end of file +}}} diff --git a/libraries/chain/webassembly/eos-vm-oc/compile_monitor.cpp b/libraries/chain/webassembly/runtimes/eos-vm-oc/compile_monitor.cpp similarity index 93% rename from libraries/chain/webassembly/eos-vm-oc/compile_monitor.cpp rename to libraries/chain/webassembly/runtimes/eos-vm-oc/compile_monitor.cpp index 67b85c5ded4..02196f1965c 100644 --- a/libraries/chain/webassembly/eos-vm-oc/compile_monitor.cpp +++ b/libraries/chain/webassembly/runtimes/eos-vm-oc/compile_monitor.cpp @@ -65,8 +65,7 @@ struct compile_monitor_session { connection_dead_signal(); return; } - - message.visit(overloaded { + std::visit(overloaded { [&, &fds=fds](const compile_wasm_message& compile) { if(fds.size() != 1) { connection_dead_signal(); @@ -85,7 +84,7 @@ struct compile_monitor_session { connection_dead_signal(); return; } - }); + }, message); read_message_from_nodeos(); }); @@ -124,8 +123,8 @@ struct compile_monitor_session { void* code_ptr = nullptr; void* mem_ptr = nullptr; try { - if(success && message.contains() && fds.size() == 2) { - code_compilation_result_message& result = message.get(); + if(success && std::holds_alternative(message) && fds.size() == 2) { + code_compilation_result_message& result = std::get(message); code_ptr = _allocator->allocate(get_size_of_fd(fds[0])); mem_ptr = _allocator->allocate(get_size_of_fd(fds[1])); @@ -201,7 +200,7 @@ struct compile_monitor { ctx.stop(); return; } - if(!message.contains() || fds.size() != 2) { + if(!std::holds_alternative(message) || fds.size() != 2) { ctx.stop(); return; } @@ -216,7 +215,7 @@ struct compile_monitor { }); write_message_with_fds(_nodeos_socket, initalize_response_message()); } - catch(const fc::exception& e) { + catch(const std::exception& e) { write_message_with_fds(_nodeos_socket, initalize_response_message{e.what()}); } catch(...) { @@ -248,6 +247,11 @@ void launch_compile_monitor(int nodeos_fd) { sigaddset(&set, SIGQUIT); sigprocmask(SIG_BLOCK, &set, nullptr); + struct sigaction sa; + sa.sa_handler = SIG_DFL; + sa.sa_flags = SA_NOCLDWAIT; + sigaction(SIGCHLD, &sa, nullptr); + int socks[2]; //0: local trampoline socket, 1: the one we give to trampoline socketpair(AF_UNIX, SOCK_SEQPACKET | SOCK_CLOEXEC, 0, socks); pid_t child = fork(); @@ -293,6 +297,8 @@ struct compile_monitor_trampoline { static compile_monitor_trampoline the_compile_monitor_trampoline; extern "C" int __real_main(int, char*[]); extern "C" int __wrap_main(int argc, char* argv[]) { + + the_compile_monitor_trampoline.start(); return __real_main(argc, argv); } @@ -318,9 +324,9 @@ wrapped_fd get_connection_to_compile_monitor(int cache_fd) { auto [success, message, fds] = read_message_with_fds(the_compile_monitor_trampoline.compile_manager_fd); EOS_ASSERT(success, misc_exception, "failed to read response from monitor process"); - EOS_ASSERT(message.contains(), misc_exception, "unexpected response from monitor process"); - EOS_ASSERT(!message.get().error_message, misc_exception, "Error message from monitor process: ${e}", ("e", *message.get().error_message)); + EOS_ASSERT(std::holds_alternative(message), misc_exception, "unexpected response from monitor process"); + EOS_ASSERT(!std::get(message).error_message, misc_exception, "Error message from monitor process: ${e}", ("e", *std::get(message).error_message)); return socket_to_monitor_session; } -}}} \ No newline at end of file +}}} diff --git a/libraries/chain/webassembly/eos-vm-oc/compile_trampoline.cpp b/libraries/chain/webassembly/runtimes/eos-vm-oc/compile_trampoline.cpp similarity index 91% rename from libraries/chain/webassembly/eos-vm-oc/compile_trampoline.cpp rename to libraries/chain/webassembly/runtimes/eos-vm-oc/compile_trampoline.cpp index 07e9ba6b086..dba86418b33 100644 --- a/libraries/chain/webassembly/eos-vm-oc/compile_trampoline.cpp +++ b/libraries/chain/webassembly/runtimes/eos-vm-oc/compile_trampoline.cpp @@ -24,6 +24,7 @@ void run_compile(wrapped_fd&& response_sock, wrapped_fd&& wasm_code) noexcept { Module module; Serialization::MemoryInputStream stream(wasm.data(), wasm.size()); + WASM::scoped_skip_checks no_check; WASM::serialize(stream, module); module.userSections.clear(); wasm_injections::wasm_binary_injection injector(module); @@ -54,7 +55,7 @@ void run_compile(wrapped_fd&& response_sock, wrapped_fd&& wasm_code) noexcept { if(module.memories.size()) result_message.starting_memory_pages = module.memories.defs.at(0).type.size.min; - std::vector prologue(memory::cb_offset); //getting the control block offset gets us as large as table+globals as possible + std::vector prologue(module.globals.defs.size() * 8); // Large enough to handle all mutable globals std::vector::iterator prologue_it = prologue.end(); //set up mutable globals @@ -85,18 +86,18 @@ void run_compile(wrapped_fd&& response_sock, wrapped_fd&& wasm_code) noexcept { int64_t func; //>= 0 means offset to code in wasm; < 0 means intrinsic call at offset address }; - if(module.tables.size()) - prologue_it -= sizeof(table_entry) * module.tables.defs[0].type.size.min; - for(const TableSegment& table_segment : module.tableSegments) { - struct table_entry* table_index_0 = (struct table_entry*)&*prologue_it; + struct table_entry* table_index_0 = (struct table_entry*)(code.code.data() + code.table_offset); + + if(static_cast(table_segment.baseOffset.i32) > module.tables.defs[0].type.size.min) + return; - if(table_segment.baseOffset.i32 > module.tables.defs[0].type.size.min) + if(static_cast(table_segment.baseOffset.i32) > module.tables.defs[0].type.size.min) return; for(Uptr i = 0; i < table_segment.indices.size(); ++i) { const Uptr function_index = table_segment.indices[i]; - const long int effective_table_index = table_segment.baseOffset.i32 + i; + const uint64_t effective_table_index = table_segment.baseOffset.i32 + i; if(effective_table_index >= module.tables.defs[0].type.size.min) return; @@ -153,8 +154,8 @@ void run_compile_trampoline(int fd) { auto [success, message, fds] = read_message_with_fds(fd); if(!success) break; - - if(!message.contains() || fds.size() != 2) { + + if(!std::holds_alternative(message) || fds.size() != 2) { std::cerr << "EOS VM OC compile trampoline got unexpected message; ignoring" << std::endl; continue; } diff --git a/libraries/chain/webassembly/runtimes/eos-vm-oc/default_real_main.cpp b/libraries/chain/webassembly/runtimes/eos-vm-oc/default_real_main.cpp new file mode 100644 index 00000000000..a6b66424e7e --- /dev/null +++ b/libraries/chain/webassembly/runtimes/eos-vm-oc/default_real_main.cpp @@ -0,0 +1,7 @@ + + +extern "C" int __real_main(int, char*[]) { + // this should never be called + return 0; +} + diff --git a/libraries/chain/webassembly/eos-vm-oc/executor.cpp b/libraries/chain/webassembly/runtimes/eos-vm-oc/executor.cpp similarity index 69% rename from libraries/chain/webassembly/eos-vm-oc/executor.cpp rename to libraries/chain/webassembly/runtimes/eos-vm-oc/executor.cpp index 1dad9744e09..ca32f8cd0e4 100644 --- a/libraries/chain/webassembly/eos-vm-oc/executor.cpp +++ b/libraries/chain/webassembly/runtimes/eos-vm-oc/executor.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include @@ -17,6 +18,7 @@ #include #include #include +#include #if defined(__has_feature) #if __has_feature(shadow_call_stack) @@ -72,7 +74,7 @@ static void segv_handler(int sig, siginfo_t* info, void* ctx) { static intrinsic grow_memory_intrinsic EOSVMOC_INTRINSIC_INIT_PRIORITY("eosvmoc_internal.grow_memory", IR::FunctionType::get(IR::ResultType::i32,{IR::ValueType::i32,IR::ValueType::i32}), (void*)&eos_vm_oc_grow_memory, - boost::hana::index_if(intrinsic_table, ::boost::hana::equal.to(BOOST_HANA_STRING("eosvmoc_internal.grow_memory"))).value() + std::integral_constant::value ); //This is effectively overriding the eosio_exit intrinsic in wasm_interface @@ -81,7 +83,7 @@ static void eosio_exit(int32_t code) { __builtin_unreachable(); } static intrinsic eosio_exit_intrinsic("env.eosio_exit", IR::FunctionType::get(IR::ResultType::none,{IR::ValueType::i32}), (void*)&eosio_exit, - boost::hana::index_if(intrinsic_table, ::boost::hana::equal.to(BOOST_HANA_STRING("env.eosio_exit"))).value() + std::integral_constant::value ); static void throw_internal_exception(const char* const s) { @@ -93,7 +95,7 @@ static void throw_internal_exception(const char* const s) { #define DEFINE_EOSVMOC_TRAP_INTRINSIC(module,name) \ void name(); \ static intrinsic name##Function EOSVMOC_INTRINSIC_INIT_PRIORITY(#module "." #name,IR::FunctionType::get(),(void*)&name, \ - boost::hana::index_if(intrinsic_table, ::boost::hana::equal.to(BOOST_HANA_STRING(#module "." #name))).value() \ + std::integral_constant::value \ ); \ void name() @@ -147,20 +149,47 @@ executor::executor(const code_cache_base& cc) { mapping_is_executable = true; } -void executor::execute(const code_descriptor& code, const memory& mem, apply_context& context) { +void executor::execute(const code_descriptor& code, memory& mem, apply_context& context) { if(mapping_is_executable == false) { mprotect(code_mapping, code_mapping_size, PROT_EXEC|PROT_READ); mapping_is_executable = true; } + uint64_t max_call_depth = eosio::chain::wasm_constraints::maximum_call_depth+1; + uint64_t max_pages = eosio::chain::wasm_constraints::maximum_linear_memory/eosio::chain::wasm_constraints::wasm_page_size; + if(context.control.is_builtin_activated(builtin_protocol_feature_t::configurable_wasm_limits)) { + const wasm_config& config = context.control.get_global_properties().wasm_configuration; + max_call_depth = config.max_call_depth; + max_pages = config.max_pages; + } + stack.reset(max_call_depth); + EOS_ASSERT(code.starting_memory_pages <= (int)max_pages, wasm_execution_error, "Initial memory out of range"); + //prepare initial memory, mutable globals, and table data if(code.starting_memory_pages > 0 ) { - arch_prctl(ARCH_SET_GS, (unsigned long*)(mem.zero_page_memory_base()+code.starting_memory_pages*memory::stride)); + uint64_t initial_page_offset = std::min(static_cast(code.starting_memory_pages), mem.size_of_memory_slice_mapping()/memory::stride - 1); + if(initial_page_offset < static_cast(code.starting_memory_pages)) { + mprotect(mem.full_page_memory_base() + initial_page_offset * eosio::chain::wasm_constraints::wasm_page_size, + (code.starting_memory_pages - initial_page_offset) * eosio::chain::wasm_constraints::wasm_page_size, PROT_READ | PROT_WRITE); + } + arch_prctl(ARCH_SET_GS, (unsigned long*)(mem.zero_page_memory_base()+initial_page_offset*memory::stride)); memset(mem.full_page_memory_base(), 0, 64u*1024u*code.starting_memory_pages); } else arch_prctl(ARCH_SET_GS, (unsigned long*)mem.zero_page_memory_base()); - memcpy(mem.full_page_memory_base() - code.initdata_prologue_size, code_mapping + code.initdata_begin, code.initdata_size); + + void* globals; + if(code.initdata_prologue_size > memory::max_prologue_size) { + globals_buffer.resize(code.initdata_prologue_size); + memcpy(globals_buffer.data(), code_mapping + code.initdata_begin, code.initdata_prologue_size); + memcpy(mem.full_page_memory_base() - memory::max_prologue_size, + code_mapping + code.initdata_begin + code.initdata_prologue_size - memory::max_prologue_size, + code.initdata_size - code.initdata_prologue_size + memory::max_prologue_size); + globals = globals_buffer.data() + globals_buffer.size(); + } else { + memcpy(mem.full_page_memory_base() - code.initdata_prologue_size, code_mapping + code.initdata_begin, code.initdata_size); + globals = mem.full_page_memory_base(); + } control_block* const cb = mem.get_control_block(); cb->magic = signal_sentinel; @@ -171,14 +200,16 @@ void executor::execute(const code_descriptor& code, const memory& mem, apply_con cb->ctx = &context; executors_exception_ptr = nullptr; cb->eptr = &executors_exception_ptr; - cb->current_call_depth_remaining = eosio::chain::wasm_constraints::maximum_call_depth+2; + cb->current_call_depth_remaining = max_call_depth + 1; cb->current_linear_memory_pages = code.starting_memory_pages; + cb->max_linear_memory_pages = max_pages; cb->first_invalid_memory_address = code.starting_memory_pages*64*1024; cb->full_linear_memory_start = (char*)mem.full_page_memory_base(); cb->jmp = &executors_sigjmp_buf; cb->bounce_buffers = &executors_bounce_buffers; cb->running_code_base = (uintptr_t)(code_mapping + code.code_begin); cb->is_running = true; + cb->globals = globals; context.trx_context.transaction_timer.set_expiration_callback([](void* user) { executor* self = (executor*)user; @@ -187,28 +218,36 @@ void executor::execute(const code_descriptor& code, const memory& mem, apply_con }, this); context.trx_context.checktime(); //catch any expiration that might have occurred before setting up callback - auto cleanup = fc::make_scoped_exit([cb, &tt=context.trx_context.transaction_timer](){ + auto cleanup = fc::make_scoped_exit([cb, &tt=context.trx_context.transaction_timer, &mem=mem](){ cb->is_running = false; cb->bounce_buffers->clear(); tt.set_expiration_callback(nullptr, nullptr); + + uint64_t base_pages = mem.size_of_memory_slice_mapping()/memory::stride - 1; + if(cb->current_linear_memory_pages > base_pages) { + mprotect(mem.full_page_memory_base() + base_pages * eosio::chain::wasm_constraints::wasm_page_size, + (cb->current_linear_memory_pages - base_pages) * eosio::chain::wasm_constraints::wasm_page_size, PROT_NONE); + } }); void(*apply_func)(uint64_t, uint64_t, uint64_t) = (void(*)(uint64_t, uint64_t, uint64_t))(cb->running_code_base + code.apply_offset); switch(sigsetjmp(*cb->jmp, 0)) { case 0: - code.start.visit(overloaded { - [&](const no_offset&) {}, - [&](const intrinsic_ordinal& i) { - void(*start_func)() = (void(*)())(*(uintptr_t*)((uintptr_t)mem.zero_page_memory_base() - memory::first_intrinsic_offset - i.ordinal*8)); - start_func(); - }, - [&](const code_offset& offs) { - void(*start_func)() = (void(*)())(cb->running_code_base + offs.offset); - start_func(); - } + stack.run([&]{ + std::visit(overloaded { + [&](const no_offset&) {}, + [&](const intrinsic_ordinal& i) { + void(*start_func)() = (void(*)())(*(uintptr_t*)((uintptr_t)mem.zero_page_memory_base() - memory::first_intrinsic_offset - i.ordinal*8)); + start_func(); + }, + [&](const code_offset& offs) { + void(*start_func)() = (void(*)())(cb->running_code_base + offs.offset); + start_func(); + } + }, code.start); + apply_func(context.get_receiver().to_uint64_t(), context.get_action().account.to_uint64_t(), context.get_action().name.to_uint64_t()); }); - apply_func(context.get_receiver().to_uint64_t(), context.get_action().account.to_uint64_t(), context.get_action().name.to_uint64_t()); break; //case 1: clean eosio_exit case EOSVMOC_EXIT_CHECKTIME_FAIL: diff --git a/libraries/chain/webassembly/eos-vm-oc/gs_seg_helpers.c b/libraries/chain/webassembly/runtimes/eos-vm-oc/gs_seg_helpers.c similarity index 53% rename from libraries/chain/webassembly/eos-vm-oc/gs_seg_helpers.c rename to libraries/chain/webassembly/runtimes/eos-vm-oc/gs_seg_helpers.c index db4f1014ec6..41841d42dd9 100644 --- a/libraries/chain/webassembly/eos-vm-oc/gs_seg_helpers.c +++ b/libraries/chain/webassembly/runtimes/eos-vm-oc/gs_seg_helpers.c @@ -2,6 +2,7 @@ #include #include +#include int arch_prctl(int code, unsigned long* addr); @@ -12,14 +13,33 @@ int32_t eos_vm_oc_grow_memory(int32_t grow, int32_t max) { uint64_t previous_page_count = cb_ptr->current_linear_memory_pages; int32_t grow_amount = grow; uint64_t max_pages = max; + if(max_pages > cb_ptr->max_linear_memory_pages) + max_pages = cb_ptr->max_linear_memory_pages; if(grow == 0) return (int32_t)cb_ptr->current_linear_memory_pages; if(previous_page_count + grow_amount > max_pages) return (int32_t)-1; + int64_t max_segments = cb_ptr->execution_thread_memory_length / EOS_VM_OC_MEMORY_STRIDE - 1; + int was_extended = previous_page_count > max_segments; + int will_be_extended = previous_page_count + grow_amount > max_segments; + char* extended_memory_start = cb_ptr->full_linear_memory_start + max_segments * 64*1024; + int64_t gs_diff; + if(will_be_extended && grow_amount > 0) { + uint64_t skip = was_extended ? previous_page_count - max_segments : 0; + gs_diff = was_extended ? 0 : max_segments - previous_page_count; + mprotect(extended_memory_start + skip * 64*1024, (grow_amount - gs_diff) * 64*1024, PROT_READ | PROT_WRITE); + } else if (was_extended && grow_amount < 0) { + uint64_t skip = will_be_extended ? previous_page_count + grow_amount - max_segments : 0; + gs_diff = will_be_extended ? 0 : previous_page_count + grow_amount - max_segments; + mprotect(extended_memory_start + skip * 64*1024, (-grow_amount + gs_diff) * 64*1024, PROT_NONE); + } else { + gs_diff = grow_amount; + } + uint64_t current_gs; arch_prctl(ARCH_GET_GS, ¤t_gs); - current_gs += grow_amount * EOS_VM_OC_MEMORY_STRIDE; + current_gs += gs_diff * EOS_VM_OC_MEMORY_STRIDE; arch_prctl(ARCH_SET_GS, (unsigned long*)current_gs); cb_ptr->current_linear_memory_pages += grow_amount; cb_ptr->first_invalid_memory_address += grow_amount*64*1024; @@ -43,4 +63,4 @@ void* eos_vm_oc_get_exception_ptr() { void* eos_vm_oc_get_bounce_buffer_list() { EOSVMOC_MEMORY_PTR_cb_ptr; return cb_ptr->bounce_buffers; -} \ No newline at end of file +} diff --git a/libraries/chain/webassembly/eos-vm-oc/intrinsic.cpp b/libraries/chain/webassembly/runtimes/eos-vm-oc/intrinsic.cpp similarity index 100% rename from libraries/chain/webassembly/eos-vm-oc/intrinsic.cpp rename to libraries/chain/webassembly/runtimes/eos-vm-oc/intrinsic.cpp diff --git a/libraries/chain/webassembly/eos-vm-oc/ipc_helpers.cpp b/libraries/chain/webassembly/runtimes/eos-vm-oc/ipc_helpers.cpp similarity index 98% rename from libraries/chain/webassembly/eos-vm-oc/ipc_helpers.cpp rename to libraries/chain/webassembly/runtimes/eos-vm-oc/ipc_helpers.cpp index cc694240957..8dcb797e5ea 100644 --- a/libraries/chain/webassembly/eos-vm-oc/ipc_helpers.cpp +++ b/libraries/chain/webassembly/runtimes/eos-vm-oc/ipc_helpers.cpp @@ -37,7 +37,7 @@ std::tuple> read_message_with_fds do { red = recvmsg(fd, &msg, 0); } while(red == -1 && errno == EINTR); - if(red < 1 || red >= sizeof(buff)) + if(red < 1 || static_cast(red) >= sizeof(buff)) return {false, message, std::move(fds)}; try { diff --git a/libraries/chain/webassembly/eos-vm-oc/llvmWARshim.cpp b/libraries/chain/webassembly/runtimes/eos-vm-oc/llvmWARshim.cpp similarity index 100% rename from libraries/chain/webassembly/eos-vm-oc/llvmWARshim.cpp rename to libraries/chain/webassembly/runtimes/eos-vm-oc/llvmWARshim.cpp diff --git a/libraries/chain/webassembly/eos-vm-oc/llvmWARshim.llvmwar b/libraries/chain/webassembly/runtimes/eos-vm-oc/llvmWARshim.llvmwar similarity index 100% rename from libraries/chain/webassembly/eos-vm-oc/llvmWARshim.llvmwar rename to libraries/chain/webassembly/runtimes/eos-vm-oc/llvmWARshim.llvmwar diff --git a/libraries/chain/webassembly/eos-vm-oc/memory.cpp b/libraries/chain/webassembly/runtimes/eos-vm-oc/memory.cpp similarity index 72% rename from libraries/chain/webassembly/eos-vm-oc/memory.cpp rename to libraries/chain/webassembly/runtimes/eos-vm-oc/memory.cpp index 6bf56f01d06..293ed229d82 100644 --- a/libraries/chain/webassembly/eos-vm-oc/memory.cpp +++ b/libraries/chain/webassembly/runtimes/eos-vm-oc/memory.cpp @@ -1,5 +1,6 @@ #include #include +#include #include @@ -10,7 +11,9 @@ namespace eosio { namespace chain { namespace eosvmoc { -memory::memory() { +memory::memory(uint64_t max_pages) { + uint64_t number_slices = max_pages + 1; + uint64_t wasm_memory_size = max_pages * wasm_constraints::wasm_page_size; int fd = syscall(SYS_memfd_create, "eosvmoc_mem", MFD_CLOEXEC); FC_ASSERT(fd >= 0, "Failed to create memory memfd"); auto cleanup_fd = fc::make_scoped_exit([&fd](){close(fd);}); @@ -40,8 +43,18 @@ memory::memory() { intrinsic_jump_table[-intrinsic.second.ordinal] = (uintptr_t)intrinsic.second.function_ptr; } +void memory::reset(uint64_t max_pages) { + uint64_t old_max_pages = mapsize / memory::total_memory_per_slice - 1; + if(max_pages == old_max_pages) return; + memory new_memory{max_pages}; + std::swap(mapbase, new_memory.mapbase); + std::swap(mapsize, new_memory.mapsize); + std::swap(zeropage_base, new_memory.zeropage_base); + std::swap(fullpage_base, new_memory.fullpage_base); +} + memory::~memory() { munmap(mapbase, mapsize); } -}}} \ No newline at end of file +}}} diff --git a/libraries/chain/webassembly/runtimes/eos-vm-oc/stack.cpp b/libraries/chain/webassembly/runtimes/eos-vm-oc/stack.cpp new file mode 100644 index 00000000000..10fed166f7b --- /dev/null +++ b/libraries/chain/webassembly/runtimes/eos-vm-oc/stack.cpp @@ -0,0 +1,27 @@ +#include +#include +#include + +using namespace eosio::chain::eosvmoc; + +void execution_stack::reset(std::size_t max_call_depth) { + if(max_call_depth > call_depth_limit) { + reset(); + std::size_t new_stack_size = max_call_depth * max_bytes_per_frame + 4*1024*1024; + void * ptr = mmap(nullptr, new_stack_size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, 0, 0); + FC_ASSERT(ptr != MAP_FAILED, "Failed to allocate wasm stack"); + mprotect(ptr, max_bytes_per_frame, PROT_NONE); + stack_top = (char*)ptr + new_stack_size; + stack_size = new_stack_size; + call_depth_limit = max_call_depth; + } +} + +void execution_stack::reset() { + if(stack_top) { + munmap((char*)stack_top - stack_size, stack_size); + stack_top = nullptr; + stack_size = 0; + call_depth_limit = 4*1024*1024 / max_bytes_per_frame; + } +} diff --git a/libraries/chain/webassembly/runtimes/eos-vm-oc/switch_stack_linux.s b/libraries/chain/webassembly/runtimes/eos-vm-oc/switch_stack_linux.s new file mode 100644 index 00000000000..37c7c1b0f65 --- /dev/null +++ b/libraries/chain/webassembly/runtimes/eos-vm-oc/switch_stack_linux.s @@ -0,0 +1,13 @@ +.file "switch_stack_linux.s" +.text +.globl eosvmoc_switch_stack +.type eosvmoc_switch_stack, @function +eosvmoc_switch_stack: + movq %rsp, -16(%rdi) + leaq -16(%rdi), %rsp + movq %rdx, %rdi + callq *%rsi + mov (%rsp), %rsp + retq +.size eosvmoc_switch_stack, .-eosvmoc_switch_stack +.section .note.GNU-stack,"",@progbits diff --git a/libraries/chain/webassembly/runtimes/eos-vm.cpp b/libraries/chain/webassembly/runtimes/eos-vm.cpp new file mode 100644 index 00000000000..1c53b62d654 --- /dev/null +++ b/libraries/chain/webassembly/runtimes/eos-vm.cpp @@ -0,0 +1,555 @@ +#include +#include +#include +#include +#include +#include +//eos-vm includes +#include +#include +#ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED +#include +#endif +#include +#include + +namespace eosio { namespace chain { namespace webassembly { namespace eos_vm_runtime { + +using namespace eosio::vm; + +namespace wasm_constraints = eosio::chain::wasm_constraints; + +namespace { + + struct checktime_watchdog { + checktime_watchdog(transaction_checktime_timer& timer) : _timer(timer) {} + template + struct guard { + guard(transaction_checktime_timer& timer, F&& func) + : _timer(timer), _func(static_cast(func)) { + _timer.set_expiration_callback(&callback, this); + if(_timer.expired) { + _func(); // it's harmless if _func is invoked twice + } + } + ~guard() { + _timer.set_expiration_callback(nullptr, nullptr); + } + static void callback(void* data) { + guard* self = static_cast(data); + self->_func(); + } + transaction_checktime_timer& _timer; + F _func; + }; + template + guard scoped_run(F&& func) { + return guard{_timer, static_cast(func)}; + } + transaction_checktime_timer& _timer; + }; +} + +// Used on setcode. Must not reject anything that WAVM accepts +// For the moment, this runs after WAVM validation, as I am not +// sure that eos-vm will replicate WAVM's parsing exactly. +struct setcode_options { + static constexpr bool forbid_export_mutable_globals = false; + static constexpr bool allow_code_after_function_end = true; + static constexpr bool allow_u32_limits_flags = true; + static constexpr bool allow_invalid_empty_local_set = true; + static constexpr bool allow_zero_blocktype = true; +}; + +void validate(const bytes& code, const whitelisted_intrinsics_type& intrinsics) { + wasm_code_ptr code_ptr((uint8_t*)code.data(), code.size()); + try { + eos_vm_null_backend_t bkend(code_ptr, code.size(), nullptr); + // check import signatures + eos_vm_host_functions_t::resolve(bkend.get_module()); + // check that the imports are all currently enabled + const auto& imports = bkend.get_module().imports; + for(std::uint32_t i = 0; i < imports.size(); ++i) { + EOS_ASSERT(std::string_view((char*)imports[i].module_str.raw(), imports[i].module_str.size()) == "env" && + is_intrinsic_whitelisted(intrinsics, std::string_view((char*)imports[i].field_str.raw(), imports[i].field_str.size())), + wasm_serialization_error, "${module}.${fn} unresolveable", + ("module", std::string((char*)imports[i].module_str.raw(), imports[i].module_str.size())) + ("fn", std::string((char*)imports[i].field_str.raw(), imports[i].field_str.size()))); + } + } catch(vm::exception& e) { + EOS_THROW(wasm_serialization_error, e.detail()); + } +} + +void validate( const bytes& code, const wasm_config& cfg, const whitelisted_intrinsics_type& intrinsics ) { + EOS_ASSERT(code.size() <= cfg.max_module_bytes, wasm_serialization_error, "Code too large"); + wasm_code_ptr code_ptr((uint8_t*)code.data(), code.size()); + try { + eos_vm_null_backend_t bkend(code_ptr, code.size(), nullptr, cfg); + // check import signatures + eos_vm_host_functions_t::resolve(bkend.get_module()); + // check that the imports are all currently enabled + const auto& imports = bkend.get_module().imports; + for(std::uint32_t i = 0; i < imports.size(); ++i) { + EOS_ASSERT(std::string_view((char*)imports[i].module_str.raw(), imports[i].module_str.size()) == "env" && + is_intrinsic_whitelisted(intrinsics, std::string_view((char*)imports[i].field_str.raw(), imports[i].field_str.size())), + wasm_serialization_error, "${module}.${fn} unresolveable", + ("module", std::string((char*)imports[i].module_str.raw(), imports[i].module_str.size())) + ("fn", std::string((char*)imports[i].field_str.raw(), imports[i].field_str.size()))); + } + // check apply + uint32_t apply_idx = bkend.get_module().get_exported_function("apply"); + EOS_ASSERT(apply_idx < std::numeric_limits::max(), wasm_serialization_error, "apply not exported"); + const vm::func_type& apply_type = bkend.get_module().get_function_type(apply_idx); + EOS_ASSERT((apply_type == vm::host_function{{vm::i64, vm::i64, vm::i64}, {}}), wasm_serialization_error, "apply has wrong type"); + } catch(vm::exception& e) { + EOS_THROW(wasm_serialization_error, e.detail()); + } +} + +// Be permissive on apply. +struct apply_options { + std::uint32_t max_pages = wasm_constraints::maximum_linear_memory/wasm_constraints::wasm_page_size; + std::uint32_t max_call_depth = wasm_constraints::maximum_call_depth+1; + static constexpr bool forbid_export_mutable_globals = false; + static constexpr bool allow_code_after_function_end = false; + static constexpr bool allow_u32_limits_flags = true; + static constexpr bool allow_invalid_empty_local_set = true; + static constexpr bool allow_zero_blocktype = true; +}; + +template +class eos_vm_instantiated_module : public wasm_instantiated_module_interface { + using backend_t = eos_vm_backend_t; + public: + + eos_vm_instantiated_module(eos_vm_runtime* runtime, std::unique_ptr mod) : + _runtime(runtime), + _instantiated_module(std::move(mod)) {} + + void apply(apply_context& context) override { + _instantiated_module->set_wasm_allocator(&context.control.get_wasm_allocator()); + _runtime->_bkend = _instantiated_module.get(); + apply_options opts; + if(context.control.is_builtin_activated(builtin_protocol_feature_t::configurable_wasm_limits)) { + const wasm_config& config = context.control.get_global_properties().wasm_configuration; + opts = {config.max_pages, config.max_call_depth}; + } + auto fn = [&]() { + eosio::chain::webassembly::interface iface(context); + _runtime->_bkend->initialize(&iface, opts); + _runtime->_bkend->call( + iface, "env", "apply", + context.get_receiver().to_uint64_t(), + context.get_action().account.to_uint64_t(), + context.get_action().name.to_uint64_t()); + }; + try { + checktime_watchdog wd(context.trx_context.transaction_timer); + _runtime->_bkend->timed_run(wd, fn); + } catch(eosio::vm::timeout_exception&) { + context.trx_context.checktime(); + } catch(eosio::vm::wasm_memory_exception& e) { + FC_THROW_EXCEPTION(wasm_execution_error, "access violation"); + } catch(eosio::vm::exception& e) { + FC_THROW_EXCEPTION(wasm_execution_error, "eos-vm system failure"); + } + _runtime->_bkend = nullptr; + } + + private: + eos_vm_runtime* _runtime; + std::unique_ptr _instantiated_module; +}; + +template +eos_vm_runtime::eos_vm_runtime() {} + +template +void eos_vm_runtime::immediately_exit_currently_running_module() { + throw wasm_exit{}; +} + +template +bool eos_vm_runtime::inject_module(IR::Module& module) { + return false; +} + +template +std::unique_ptr eos_vm_runtime::instantiate_module(const char* code_bytes, size_t code_size, std::vector, + const digest_type&, const uint8_t&, const uint8_t&) { + + using backend_t = eos_vm_backend_t; + try { + wasm_code_ptr code((uint8_t*)code_bytes, code_size); + apply_options options = { .max_pages = 65536, + .max_call_depth = 0 }; + std::unique_ptr bkend = std::make_unique(code, code_size, nullptr, options); + eos_vm_host_functions_t::resolve(bkend->get_module()); + return std::make_unique>(this, std::move(bkend)); + } catch(eosio::vm::exception& e) { + FC_THROW_EXCEPTION(wasm_execution_error, "Error building eos-vm interp: ${e}", ("e", e.what())); + } +} + +template class eos_vm_runtime; +template class eos_vm_runtime; + +} + +template +struct host_function_registrator { + template + constexpr host_function_registrator(Mod mod_name, Name fn_name) { + using rhf_t = eos_vm_host_functions_t; + rhf_t::add(mod_name.c_str(), fn_name.c_str()); +#ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED + constexpr bool is_injected = (Mod() == BOOST_HANA_STRING(EOSIO_INJECTED_MODULE_NAME)); + eosvmoc::register_eosvm_oc>( + mod_name + BOOST_HANA_STRING(".") + fn_name); +#endif + } +}; + +#define REGISTER_INJECTED_HOST_FUNCTION(NAME, ...) \ + static host_function_registrator<&interface::NAME, ##__VA_ARGS__> NAME##_registrator_impl() { \ + return {BOOST_HANA_STRING(EOSIO_INJECTED_MODULE_NAME), BOOST_HANA_STRING(#NAME)}; \ + } \ + inline static auto NAME##_registrator = NAME##_registrator_impl(); + +#define REGISTER_HOST_FUNCTION(NAME, ...) \ + static host_function_registrator<&interface::NAME, core_precondition, context_aware_check, ##__VA_ARGS__> \ + NAME##_registrator_impl() { \ + return {BOOST_HANA_STRING("env"), BOOST_HANA_STRING(#NAME)}; \ + } \ + inline static auto NAME##_registrator = NAME##_registrator_impl(); + +#define REGISTER_CF_HOST_FUNCTION(NAME, ...) \ + static host_function_registrator<&interface::NAME, core_precondition, ##__VA_ARGS__> NAME##_registrator_impl() { \ + return {BOOST_HANA_STRING("env"), BOOST_HANA_STRING(#NAME)}; \ + } \ + inline static auto NAME##_registrator = NAME##_registrator_impl(); + +#define REGISTER_LEGACY_HOST_FUNCTION(NAME, ...) \ + static host_function_registrator<&interface::NAME, legacy_static_check_wl_args, context_aware_check, ##__VA_ARGS__> \ + NAME##_registrator_impl() { \ + return {BOOST_HANA_STRING("env"), BOOST_HANA_STRING(#NAME)}; \ + } \ + inline static auto NAME##_registrator = NAME##_registrator_impl(); + +#define REGISTER_LEGACY_CF_HOST_FUNCTION(NAME, ...) \ + static host_function_registrator<&interface::NAME, legacy_static_check_wl_args, ##__VA_ARGS__> \ + NAME##_registrator_impl() { \ + return {BOOST_HANA_STRING("env"), BOOST_HANA_STRING(#NAME)}; \ + } \ + inline static auto NAME##_registrator = NAME##_registrator_impl(); + +#define REGISTER_LEGACY_CF_ONLY_HOST_FUNCTION(NAME, ...) \ + static host_function_registrator<&interface::NAME, legacy_static_check_wl_args, context_free_check, ##__VA_ARGS__> \ + NAME##_registrator_impl() { \ + return {BOOST_HANA_STRING("env"), BOOST_HANA_STRING(#NAME)}; \ + } \ + inline static auto NAME##_registrator = NAME##_registrator_impl(); + +// context free api +REGISTER_LEGACY_CF_ONLY_HOST_FUNCTION(get_context_free_data) + +// privileged api +REGISTER_HOST_FUNCTION(is_feature_active, privileged_check); +REGISTER_HOST_FUNCTION(activate_feature, privileged_check); +REGISTER_LEGACY_HOST_FUNCTION(preactivate_feature, privileged_check); +REGISTER_HOST_FUNCTION(set_resource_limits, privileged_check); +REGISTER_LEGACY_HOST_FUNCTION(get_resource_limits, privileged_check); +REGISTER_HOST_FUNCTION(set_resource_limit, privileged_check); +REGISTER_HOST_FUNCTION(get_resource_limit, privileged_check); +REGISTER_HOST_FUNCTION(get_wasm_parameters_packed, privileged_check); +REGISTER_HOST_FUNCTION(set_wasm_parameters_packed, privileged_check); +REGISTER_LEGACY_HOST_FUNCTION(set_proposed_producers, privileged_check); +REGISTER_LEGACY_HOST_FUNCTION(set_proposed_producers_ex, privileged_check); +REGISTER_LEGACY_HOST_FUNCTION(get_blockchain_parameters_packed, privileged_check); +REGISTER_LEGACY_HOST_FUNCTION(set_blockchain_parameters_packed, privileged_check); +REGISTER_HOST_FUNCTION(get_parameters_packed, privileged_check); +REGISTER_HOST_FUNCTION(set_parameters_packed, privileged_check); +REGISTER_HOST_FUNCTION(get_kv_parameters_packed, privileged_check); +REGISTER_HOST_FUNCTION(set_kv_parameters_packed, privileged_check); +REGISTER_HOST_FUNCTION(is_privileged, privileged_check); +REGISTER_HOST_FUNCTION(set_privileged, privileged_check); + +// softfloat api +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_add); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_sub); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_div); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_mul); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_min); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_max); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_copysign); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_abs); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_neg); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_sqrt); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_ceil); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_floor); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_trunc); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_nearest); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_eq); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_ne); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_lt); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_le); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_gt); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_ge); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_add); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_sub); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_div); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_mul); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_min); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_max); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_copysign); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_abs); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_neg); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_sqrt); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_ceil); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_floor); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_trunc); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_nearest); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_eq); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_ne); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_lt); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_le); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_gt); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_ge); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_promote); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_demote); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_trunc_i32s); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_trunc_i32s); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_trunc_i32u); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_trunc_i32u); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_trunc_i64s); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_trunc_i64s); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f32_trunc_i64u); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_f64_trunc_i64u); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_i32_to_f32); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_i64_to_f32); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_ui32_to_f32); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_ui64_to_f32); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_i32_to_f64); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_i64_to_f64); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_ui32_to_f64); +REGISTER_INJECTED_HOST_FUNCTION(_eosio_ui64_to_f64); + +// producer api +REGISTER_LEGACY_HOST_FUNCTION(get_active_producers); + +// crypto api +REGISTER_LEGACY_CF_HOST_FUNCTION(assert_recover_key); +REGISTER_LEGACY_CF_HOST_FUNCTION(recover_key); +REGISTER_LEGACY_CF_HOST_FUNCTION(assert_sha256); +REGISTER_LEGACY_CF_HOST_FUNCTION(assert_sha1); +REGISTER_LEGACY_CF_HOST_FUNCTION(assert_sha512); +REGISTER_LEGACY_CF_HOST_FUNCTION(assert_ripemd160); +REGISTER_LEGACY_CF_HOST_FUNCTION(sha256); +REGISTER_LEGACY_CF_HOST_FUNCTION(sha1); +REGISTER_LEGACY_CF_HOST_FUNCTION(sha512); +REGISTER_LEGACY_CF_HOST_FUNCTION(ripemd160); + +// permission api +REGISTER_LEGACY_HOST_FUNCTION(check_transaction_authorization); +REGISTER_LEGACY_HOST_FUNCTION(check_permission_authorization); +REGISTER_HOST_FUNCTION(get_permission_last_used); +REGISTER_HOST_FUNCTION(get_account_creation_time); + +// authorization api +REGISTER_HOST_FUNCTION(require_auth); +REGISTER_HOST_FUNCTION(require_auth2); +REGISTER_HOST_FUNCTION(has_auth); +REGISTER_HOST_FUNCTION(require_recipient); +REGISTER_HOST_FUNCTION(is_account); + +// system api +REGISTER_HOST_FUNCTION(current_time); +REGISTER_HOST_FUNCTION(publication_time); +REGISTER_LEGACY_HOST_FUNCTION(is_feature_activated); +REGISTER_HOST_FUNCTION(get_sender); + +// context-free system api +REGISTER_CF_HOST_FUNCTION(abort) +REGISTER_LEGACY_CF_HOST_FUNCTION(eosio_assert) +REGISTER_LEGACY_CF_HOST_FUNCTION(eosio_assert_message) +REGISTER_CF_HOST_FUNCTION(eosio_assert_code) +REGISTER_CF_HOST_FUNCTION(eosio_exit) + +// action api +REGISTER_LEGACY_CF_HOST_FUNCTION(read_action_data); +REGISTER_CF_HOST_FUNCTION(action_data_size); +REGISTER_CF_HOST_FUNCTION(current_receiver); +REGISTER_HOST_FUNCTION(set_action_return_value); + +// console api +REGISTER_LEGACY_CF_HOST_FUNCTION(prints); +REGISTER_LEGACY_CF_HOST_FUNCTION(prints_l); +REGISTER_CF_HOST_FUNCTION(printi); +REGISTER_CF_HOST_FUNCTION(printui); +REGISTER_LEGACY_CF_HOST_FUNCTION(printi128); +REGISTER_LEGACY_CF_HOST_FUNCTION(printui128); +REGISTER_CF_HOST_FUNCTION(printsf); +REGISTER_CF_HOST_FUNCTION(printdf); +REGISTER_LEGACY_CF_HOST_FUNCTION(printqf); +REGISTER_CF_HOST_FUNCTION(printn); +REGISTER_LEGACY_CF_HOST_FUNCTION(printhex); + +// database api +// primary index api +REGISTER_LEGACY_HOST_FUNCTION(db_store_i64); +REGISTER_LEGACY_HOST_FUNCTION(db_update_i64); +REGISTER_HOST_FUNCTION(db_remove_i64); +REGISTER_LEGACY_HOST_FUNCTION(db_get_i64); +REGISTER_LEGACY_HOST_FUNCTION(db_next_i64); +REGISTER_LEGACY_HOST_FUNCTION(db_previous_i64); +REGISTER_HOST_FUNCTION(db_find_i64); +REGISTER_HOST_FUNCTION(db_lowerbound_i64); +REGISTER_HOST_FUNCTION(db_upperbound_i64); +REGISTER_HOST_FUNCTION(db_end_i64); + +// uint64_t secondary index api +REGISTER_LEGACY_HOST_FUNCTION(db_idx64_store); +REGISTER_LEGACY_HOST_FUNCTION(db_idx64_update); +REGISTER_HOST_FUNCTION(db_idx64_remove); +REGISTER_LEGACY_HOST_FUNCTION(db_idx64_find_secondary); +REGISTER_LEGACY_HOST_FUNCTION(db_idx64_find_primary); +REGISTER_LEGACY_HOST_FUNCTION(db_idx64_lowerbound); +REGISTER_LEGACY_HOST_FUNCTION(db_idx64_upperbound); +REGISTER_HOST_FUNCTION(db_idx64_end); +REGISTER_LEGACY_HOST_FUNCTION(db_idx64_next); +REGISTER_LEGACY_HOST_FUNCTION(db_idx64_previous); + +// uint128_t secondary index api +REGISTER_LEGACY_HOST_FUNCTION(db_idx128_store); +REGISTER_LEGACY_HOST_FUNCTION(db_idx128_update); +REGISTER_HOST_FUNCTION(db_idx128_remove); +REGISTER_LEGACY_HOST_FUNCTION(db_idx128_find_secondary); +REGISTER_LEGACY_HOST_FUNCTION(db_idx128_find_primary); +REGISTER_LEGACY_HOST_FUNCTION(db_idx128_lowerbound); +REGISTER_LEGACY_HOST_FUNCTION(db_idx128_upperbound); +REGISTER_HOST_FUNCTION(db_idx128_end); +REGISTER_LEGACY_HOST_FUNCTION(db_idx128_next); +REGISTER_LEGACY_HOST_FUNCTION(db_idx128_previous); + +// 256-bit secondary index api +REGISTER_LEGACY_HOST_FUNCTION(db_idx256_store); +REGISTER_LEGACY_HOST_FUNCTION(db_idx256_update); +REGISTER_HOST_FUNCTION(db_idx256_remove); +REGISTER_LEGACY_HOST_FUNCTION(db_idx256_find_secondary); +REGISTER_LEGACY_HOST_FUNCTION(db_idx256_find_primary); +REGISTER_LEGACY_HOST_FUNCTION(db_idx256_lowerbound); +REGISTER_LEGACY_HOST_FUNCTION(db_idx256_upperbound); +REGISTER_HOST_FUNCTION(db_idx256_end); +REGISTER_LEGACY_HOST_FUNCTION(db_idx256_next); +REGISTER_LEGACY_HOST_FUNCTION(db_idx256_previous); + +// double secondary index api +REGISTER_LEGACY_HOST_FUNCTION(db_idx_double_store, is_nan_check); +REGISTER_LEGACY_HOST_FUNCTION(db_idx_double_update, is_nan_check); +REGISTER_HOST_FUNCTION(db_idx_double_remove); +REGISTER_LEGACY_HOST_FUNCTION(db_idx_double_find_secondary, is_nan_check); +REGISTER_LEGACY_HOST_FUNCTION(db_idx_double_find_primary); +REGISTER_LEGACY_HOST_FUNCTION(db_idx_double_lowerbound, is_nan_check); +REGISTER_LEGACY_HOST_FUNCTION(db_idx_double_upperbound, is_nan_check); +REGISTER_HOST_FUNCTION(db_idx_double_end); +REGISTER_LEGACY_HOST_FUNCTION(db_idx_double_next); +REGISTER_LEGACY_HOST_FUNCTION(db_idx_double_previous); + +// long double secondary index api +REGISTER_LEGACY_HOST_FUNCTION(db_idx_long_double_store, is_nan_check); +REGISTER_LEGACY_HOST_FUNCTION(db_idx_long_double_update, is_nan_check); +REGISTER_HOST_FUNCTION(db_idx_long_double_remove); +REGISTER_LEGACY_HOST_FUNCTION(db_idx_long_double_find_secondary, is_nan_check); +REGISTER_LEGACY_HOST_FUNCTION(db_idx_long_double_find_primary); +REGISTER_LEGACY_HOST_FUNCTION(db_idx_long_double_lowerbound, is_nan_check); +REGISTER_LEGACY_HOST_FUNCTION(db_idx_long_double_upperbound, is_nan_check); +REGISTER_HOST_FUNCTION(db_idx_long_double_end); +REGISTER_LEGACY_HOST_FUNCTION(db_idx_long_double_next); +REGISTER_LEGACY_HOST_FUNCTION(db_idx_long_double_previous); + +// kv database api +REGISTER_HOST_FUNCTION(kv_erase); +REGISTER_HOST_FUNCTION(kv_set); +REGISTER_HOST_FUNCTION(kv_get); +REGISTER_HOST_FUNCTION(kv_get_data); +REGISTER_HOST_FUNCTION(kv_it_create); +REGISTER_HOST_FUNCTION(kv_it_destroy); +REGISTER_HOST_FUNCTION(kv_it_status); +REGISTER_HOST_FUNCTION(kv_it_compare); +REGISTER_HOST_FUNCTION(kv_it_key_compare); +REGISTER_HOST_FUNCTION(kv_it_move_to_end); +REGISTER_HOST_FUNCTION(kv_it_next); +REGISTER_HOST_FUNCTION(kv_it_prev); +REGISTER_HOST_FUNCTION(kv_it_lower_bound); +REGISTER_HOST_FUNCTION(kv_it_key); +REGISTER_HOST_FUNCTION(kv_it_value); + +// memory api +REGISTER_LEGACY_CF_HOST_FUNCTION(memcpy); +REGISTER_LEGACY_CF_HOST_FUNCTION(memmove); +REGISTER_LEGACY_CF_HOST_FUNCTION(memcmp); +REGISTER_LEGACY_CF_HOST_FUNCTION(memset); + +// transaction api +REGISTER_LEGACY_HOST_FUNCTION(send_inline); +REGISTER_LEGACY_HOST_FUNCTION(send_context_free_inline); +REGISTER_LEGACY_HOST_FUNCTION(send_deferred); +REGISTER_LEGACY_HOST_FUNCTION(cancel_deferred); + +// context-free transaction api +REGISTER_LEGACY_CF_HOST_FUNCTION(read_transaction); +REGISTER_CF_HOST_FUNCTION(transaction_size); +REGISTER_CF_HOST_FUNCTION(expiration); +REGISTER_CF_HOST_FUNCTION(tapos_block_num); +REGISTER_CF_HOST_FUNCTION(tapos_block_prefix); +REGISTER_LEGACY_CF_HOST_FUNCTION(get_action); + +// compiler builtins api +REGISTER_LEGACY_CF_HOST_FUNCTION(__ashlti3); +REGISTER_LEGACY_CF_HOST_FUNCTION(__ashrti3); +REGISTER_LEGACY_CF_HOST_FUNCTION(__lshlti3); +REGISTER_LEGACY_CF_HOST_FUNCTION(__lshrti3); +REGISTER_LEGACY_CF_HOST_FUNCTION(__divti3); +REGISTER_LEGACY_CF_HOST_FUNCTION(__udivti3); +REGISTER_LEGACY_CF_HOST_FUNCTION(__multi3); +REGISTER_LEGACY_CF_HOST_FUNCTION(__modti3); +REGISTER_LEGACY_CF_HOST_FUNCTION(__umodti3); +REGISTER_LEGACY_CF_HOST_FUNCTION(__addtf3); +REGISTER_LEGACY_CF_HOST_FUNCTION(__subtf3); +REGISTER_LEGACY_CF_HOST_FUNCTION(__multf3); +REGISTER_LEGACY_CF_HOST_FUNCTION(__divtf3); +REGISTER_LEGACY_CF_HOST_FUNCTION(__negtf2); +REGISTER_LEGACY_CF_HOST_FUNCTION(__extendsftf2); +REGISTER_LEGACY_CF_HOST_FUNCTION(__extenddftf2); +REGISTER_CF_HOST_FUNCTION(__trunctfdf2); +REGISTER_CF_HOST_FUNCTION(__trunctfsf2); +REGISTER_CF_HOST_FUNCTION(__fixtfsi); +REGISTER_CF_HOST_FUNCTION(__fixtfdi); +REGISTER_LEGACY_CF_HOST_FUNCTION(__fixtfti); +REGISTER_CF_HOST_FUNCTION(__fixunstfsi); +REGISTER_CF_HOST_FUNCTION(__fixunstfdi); +REGISTER_LEGACY_CF_HOST_FUNCTION(__fixunstfti); +REGISTER_LEGACY_CF_HOST_FUNCTION(__fixsfti); +REGISTER_LEGACY_CF_HOST_FUNCTION(__fixdfti); +REGISTER_LEGACY_CF_HOST_FUNCTION(__fixunssfti); +REGISTER_LEGACY_CF_HOST_FUNCTION(__fixunsdfti); +REGISTER_CF_HOST_FUNCTION(__floatsidf); +REGISTER_LEGACY_CF_HOST_FUNCTION(__floatsitf); +REGISTER_LEGACY_CF_HOST_FUNCTION(__floatditf); +REGISTER_LEGACY_CF_HOST_FUNCTION(__floatunsitf); +REGISTER_LEGACY_CF_HOST_FUNCTION(__floatunditf); +REGISTER_CF_HOST_FUNCTION(__floattidf); +REGISTER_CF_HOST_FUNCTION(__floatuntidf); +REGISTER_CF_HOST_FUNCTION(__cmptf2); +REGISTER_CF_HOST_FUNCTION(__eqtf2); +REGISTER_CF_HOST_FUNCTION(__netf2); +REGISTER_CF_HOST_FUNCTION(__getf2); +REGISTER_CF_HOST_FUNCTION(__gttf2); +REGISTER_CF_HOST_FUNCTION(__letf2); +REGISTER_CF_HOST_FUNCTION(__lttf2); +REGISTER_CF_HOST_FUNCTION(__unordtf2); + +} // namespace webassembly +} // namespace chain +} // namespace eosio diff --git a/libraries/chain/webassembly/softfloat.cpp b/libraries/chain/webassembly/softfloat.cpp new file mode 100644 index 00000000000..a5610e8280d --- /dev/null +++ b/libraries/chain/webassembly/softfloat.cpp @@ -0,0 +1,452 @@ +#include +#include +#include + +namespace eosio { namespace chain { namespace webassembly { + static constexpr uint32_t inv_float_eps = 0x4B000000; + static constexpr uint64_t inv_double_eps = 0x4330000000000000; + + // float binops + float interface::_eosio_f32_add( float a, float b ) const { + float32_t r = ::f32_add( to_softfloat32(a), to_softfloat32(b) ); + float ret; + std::memcpy((char*)&ret, (char*)&r, sizeof(ret)); + return ret; + } + float interface::_eosio_f32_sub( float a, float b ) const { + float32_t r = ::f32_sub( to_softfloat32(a), to_softfloat32(b) ); + float ret; + std::memcpy((char*)&ret, (char*)&r, sizeof(ret)); + return ret; + } + float interface::_eosio_f32_div( float a, float b ) const { + float32_t r = ::f32_div( to_softfloat32(a), to_softfloat32(b) ); + float ret; + std::memcpy((char*)&ret, (char*)&r, sizeof(ret)); + return ret; + } + float interface::_eosio_f32_mul( float a, float b ) const { + float32_t r = ::f32_mul( to_softfloat32(a), to_softfloat32(b) ); + float ret; + std::memcpy((char*)&ret, (char*)&r, sizeof(ret)); + return ret; + } + float interface::_eosio_f32_min( float af, float bf ) const { + float32_t a = to_softfloat32(af); + float32_t b = to_softfloat32(bf); + if (is_nan(a)) { + return af; + } + if (is_nan(b)) { + return bf; + } + if ( f32_sign_bit(a) != f32_sign_bit(b) ) { + return f32_sign_bit(a) ? af : bf; + } + return ::f32_lt(a,b) ? af : bf; + } + float interface::_eosio_f32_max( float af, float bf ) const { + float32_t a = to_softfloat32(af); + float32_t b = to_softfloat32(bf); + if (is_nan(a)) { + return af; + } + if (is_nan(b)) { + return bf; + } + if ( f32_sign_bit(a) != f32_sign_bit(b) ) { + return f32_sign_bit(a) ? bf : af; + } + return ::f32_lt( a, b ) ? bf : af; + } + float interface::_eosio_f32_copysign( float af, float bf ) const { + float32_t a = to_softfloat32(af); + float32_t b = to_softfloat32(bf); + uint32_t sign_of_b = b.v >> 31; + a.v &= ~(1 << 31); // clear the sign bit + a.v = a.v | (sign_of_b << 31); // add the sign of b + return from_softfloat32(a); + } + // float unops + float interface::_eosio_f32_abs( float af ) const { + float32_t a = to_softfloat32(af); + a.v &= ~(1 << 31); + return from_softfloat32(a); + } + float interface::_eosio_f32_neg( float af ) const { + float32_t a = to_softfloat32(af); + uint32_t sign = a.v >> 31; + a.v &= ~(1 << 31); + a.v |= (!sign << 31); + return from_softfloat32(a); + } + float interface::_eosio_f32_sqrt( float a ) const { + float32_t ret = ::f32_sqrt( to_softfloat32(a) ); + return from_softfloat32(ret); + } + // ceil, floor, trunc and nearest are lifted from libc + float interface::_eosio_f32_ceil( float af ) const { + float32_t a = to_softfloat32(af); + int e = (int)(a.v >> 23 & 0xFF) - 0X7F; + uint32_t m; + if (e >= 23) + return af; + if (e >= 0) { + m = 0x007FFFFF >> e; + if ((a.v & m) == 0) + return af; + if (a.v >> 31 == 0) + a.v += m; + a.v &= ~m; + } else { + if (a.v >> 31) + a.v = 0x80000000; // return -0.0f + else if (a.v << 1) + a.v = 0x3F800000; // return 1.0f + } + + return from_softfloat32(a); + } + float interface::_eosio_f32_floor( float af ) const { + float32_t a = to_softfloat32(af); + int e = (int)(a.v >> 23 & 0xFF) - 0X7F; + uint32_t m; + if (e >= 23) + return af; + if (e >= 0) { + m = 0x007FFFFF >> e; + if ((a.v & m) == 0) + return af; + if (a.v >> 31) + a.v += m; + a.v &= ~m; + } else { + if (a.v >> 31 == 0) + a.v = 0; + else if (a.v << 1) + a.v = 0xBF800000; // return -1.0f + } + return from_softfloat32(a); + } + float interface::_eosio_f32_trunc( float af ) const { + float32_t a = to_softfloat32(af); + int e = (int)(a.v >> 23 & 0xff) - 0x7f + 9; + uint32_t m; + if (e >= 23 + 9) + return af; + if (e < 9) + e = 1; + m = -1U >> e; + if ((a.v & m) == 0) + return af; + a.v &= ~m; + return from_softfloat32(a); + } + float interface::_eosio_f32_nearest( float af ) const { + float32_t a = to_softfloat32(af); + int e = a.v>>23 & 0xff; + int s = a.v>>31; + float32_t y; + if (e >= 0x7f+23) + return af; + if (s) + y = ::f32_add( ::f32_sub( a, float32_t{inv_float_eps} ), float32_t{inv_float_eps} ); + else + y = ::f32_sub( ::f32_add( a, float32_t{inv_float_eps} ), float32_t{inv_float_eps} ); + if (::f32_eq( y, {0} ) ) + return s ? -0.0f : 0.0f; + return from_softfloat32(y); + } + + // float relops + bool interface::_eosio_f32_eq( float a, float b ) const { return ::f32_eq( to_softfloat32(a), to_softfloat32(b) ); } + bool interface::_eosio_f32_ne( float a, float b ) const { return !::f32_eq( to_softfloat32(a), to_softfloat32(b) ); } + bool interface::_eosio_f32_lt( float a, float b ) const { return ::f32_lt( to_softfloat32(a), to_softfloat32(b) ); } + bool interface::_eosio_f32_le( float a, float b ) const { return ::f32_le( to_softfloat32(a), to_softfloat32(b) ); } + bool interface::_eosio_f32_gt( float af, float bf ) const { + float32_t a = to_softfloat32(af); + float32_t b = to_softfloat32(bf); + if (is_nan(a)) + return false; + if (is_nan(b)) + return false; + return !::f32_le( a, b ); + } + bool interface::_eosio_f32_ge( float af, float bf ) const { + float32_t a = to_softfloat32(af); + float32_t b = to_softfloat32(bf); + if (is_nan(a)) + return false; + if (is_nan(b)) + return false; + return !::f32_lt( a, b ); + } + + // double binops + double interface::_eosio_f64_add( double a, double b ) const { + float64_t ret = ::f64_add( to_softfloat64(a), to_softfloat64(b) ); + return from_softfloat64(ret); + } + double interface::_eosio_f64_sub( double a, double b ) const { + float64_t ret = ::f64_sub( to_softfloat64(a), to_softfloat64(b) ); + return from_softfloat64(ret); + } + double interface::_eosio_f64_div( double a, double b ) const { + float64_t ret = ::f64_div( to_softfloat64(a), to_softfloat64(b) ); + return from_softfloat64(ret); + } + double interface::_eosio_f64_mul( double a, double b ) const { + float64_t ret = ::f64_mul( to_softfloat64(a), to_softfloat64(b) ); + return from_softfloat64(ret); + } + double interface::_eosio_f64_min( double af, double bf ) const { + float64_t a = to_softfloat64(af); + float64_t b = to_softfloat64(bf); + if (is_nan(a)) + return af; + if (is_nan(b)) + return bf; + if (f64_sign_bit(a) != f64_sign_bit(b)) + return f64_sign_bit(a) ? af : bf; + return ::f64_lt( a, b ) ? af : bf; + } + double interface::_eosio_f64_max( double af, double bf ) const { + float64_t a = to_softfloat64(af); + float64_t b = to_softfloat64(bf); + if (is_nan(a)) + return af; + if (is_nan(b)) + return bf; + if (f64_sign_bit(a) != f64_sign_bit(b)) + return f64_sign_bit(a) ? bf : af; + return ::f64_lt( a, b ) ? bf : af; + } + double interface::_eosio_f64_copysign( double af, double bf ) const { + float64_t a = to_softfloat64(af); + float64_t b = to_softfloat64(bf); + uint64_t sign_of_b = b.v >> 63; + a.v &= ~(uint64_t(1) << 63); // clear the sign bit + a.v = a.v | (sign_of_b << 63); // add the sign of b + return from_softfloat64(a); + } + + // double unops + double interface::_eosio_f64_abs( double af ) const { + float64_t a = to_softfloat64(af); + a.v &= ~(uint64_t(1) << 63); + return from_softfloat64(a); + } + double interface::_eosio_f64_neg( double af ) const { + float64_t a = to_softfloat64(af); + uint64_t sign = a.v >> 63; + a.v &= ~(uint64_t(1) << 63); + a.v |= (uint64_t(!sign) << 63); + return from_softfloat64(a); + } + double interface::_eosio_f64_sqrt( double a ) const { + float64_t ret = ::f64_sqrt( to_softfloat64(a) ); + return from_softfloat64(ret); + } + // ceil, floor, trunc and nearest are lifted from libc + double interface::_eosio_f64_ceil( double af ) const { + float64_t a = to_softfloat64( af ); + float64_t ret; + int e = a.v >> 52 & 0x7ff; + float64_t y; + if (e >= 0x3ff+52 || ::f64_eq( a, { 0 } )) + return af; + /* y = int(x) - x, where int(x) is an integer neighbor of x */ + if (a.v >> 63) + y = ::f64_sub( ::f64_add( ::f64_sub( a, float64_t{inv_double_eps} ), float64_t{inv_double_eps} ), a ); + else + y = ::f64_sub( ::f64_sub( ::f64_add( a, float64_t{inv_double_eps} ), float64_t{inv_double_eps} ), a ); + /* special case because of non-nearest rounding modes */ + if (e <= 0x3ff-1) { + return a.v >> 63 ? -0.0 : 1.0; //float64_t{0x8000000000000000} : float64_t{0xBE99999A3F800000}; //either -0.0 or 1 + } + if (::f64_lt( y, to_softfloat64(0) )) { + ret = ::f64_add( ::f64_add( a, y ), to_softfloat64(1) ); // 0xBE99999A3F800000 } ); // plus 1 + return from_softfloat64(ret); + } + ret = ::f64_add( a, y ); + return from_softfloat64(ret); + } + double interface::_eosio_f64_floor( double af ) const { + float64_t a = to_softfloat64( af ); + float64_t ret; + int e = a.v >> 52 & 0x7FF; + float64_t y; + double de = 1/DBL_EPSILON; + if ( a.v == 0x8000000000000000) { + return af; + } + if (e >= 0x3FF+52 || a.v == 0) { + return af; + } + if (a.v >> 63) + y = ::f64_sub( ::f64_add( ::f64_sub( a, float64_t{inv_double_eps} ), float64_t{inv_double_eps} ), a ); + else + y = ::f64_sub( ::f64_sub( ::f64_add( a, float64_t{inv_double_eps} ), float64_t{inv_double_eps} ), a ); + if (e <= 0x3FF-1) { + return a.v>>63 ? -1.0 : 0.0; //float64_t{0xBFF0000000000000} : float64_t{0}; // -1 or 0 + } + if ( !::f64_le( y, float64_t{0} ) ) { + ret = ::f64_sub( ::f64_add(a,y), to_softfloat64(1.0)); + return from_softfloat64(ret); + } + ret = ::f64_add( a, y ); + return from_softfloat64(ret); + } + double interface::_eosio_f64_trunc( double af ) const { + float64_t a = to_softfloat64( af ); + int e = (int)(a.v >> 52 & 0x7ff) - 0x3ff + 12; + uint64_t m; + if (e >= 52 + 12) + return af; + if (e < 12) + e = 1; + m = -1ULL >> e; + if ((a.v & m) == 0) + return af; + a.v &= ~m; + return from_softfloat64(a); + } + + double interface::_eosio_f64_nearest( double af ) const { + float64_t a = to_softfloat64( af ); + int e = (a.v >> 52 & 0x7FF); + int s = a.v >> 63; + float64_t y; + if ( e >= 0x3FF+52 ) + return af; + if ( s ) + y = ::f64_add( ::f64_sub( a, float64_t{inv_double_eps} ), float64_t{inv_double_eps} ); + else + y = ::f64_sub( ::f64_add( a, float64_t{inv_double_eps} ), float64_t{inv_double_eps} ); + if ( ::f64_eq( y, float64_t{0} ) ) + return s ? -0.0 : 0.0; + return from_softfloat64(y); + } + + // double relops + bool interface::_eosio_f64_eq( double a, double b ) const { return ::f64_eq( to_softfloat64(a), to_softfloat64(b) ); } + bool interface::_eosio_f64_ne( double a, double b ) const { return !::f64_eq( to_softfloat64(a), to_softfloat64(b) ); } + bool interface::_eosio_f64_lt( double a, double b ) const { return ::f64_lt( to_softfloat64(a), to_softfloat64(b) ); } + bool interface::_eosio_f64_le( double a, double b ) const { return ::f64_le( to_softfloat64(a), to_softfloat64(b) ); } + bool interface::_eosio_f64_gt( double af, double bf ) const { + float64_t a = to_softfloat64(af); + float64_t b = to_softfloat64(bf); + if (is_nan(a)) + return false; + if (is_nan(b)) + return false; + return !::f64_le( a, b ); + } + bool interface::_eosio_f64_ge( double af, double bf ) const { + float64_t a = to_softfloat64(af); + float64_t b = to_softfloat64(bf); + if (is_nan(a)) + return false; + if (is_nan(b)) + return false; + return !::f64_lt( a, b ); + } + + // float and double conversions + double interface::_eosio_f32_promote( float a ) const { + return from_softfloat64(f32_to_f64( to_softfloat32(a)) ); + } + float interface::_eosio_f64_demote( double a ) const { + return from_softfloat32(f64_to_f32( to_softfloat64(a)) ); + } + int32_t interface::_eosio_f32_trunc_i32s( float af ) const { + float32_t a = to_softfloat32(af); + if (_eosio_f32_ge(af, 2147483648.0f) || _eosio_f32_lt(af, -2147483648.0f)) + FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f32.convert_s/i32 overflow" ); + + if (is_nan(a)) + FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f32.convert_s/i32 unrepresentable"); + return f32_to_i32( to_softfloat32(_eosio_f32_trunc( af )), 0, false ); + } + int32_t interface::_eosio_f64_trunc_i32s( double af ) const { + float64_t a = to_softfloat64(af); + if (_eosio_f64_ge(af, 2147483648.0) || _eosio_f64_lt(af, -2147483648.0)) + FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f64.convert_s/i32 overflow"); + if (is_nan(a)) + FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f64.convert_s/i32 unrepresentable"); + return f64_to_i32( to_softfloat64(_eosio_f64_trunc( af )), 0, false ); + } + uint32_t interface::_eosio_f32_trunc_i32u( float af ) const { + float32_t a = to_softfloat32(af); + if (_eosio_f32_ge(af, 4294967296.0f) || _eosio_f32_le(af, -1.0f)) + FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f32.convert_u/i32 overflow"); + if (is_nan(a)) + FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f32.convert_u/i32 unrepresentable"); + return f32_to_ui32( to_softfloat32(_eosio_f32_trunc( af )), 0, false ); + } + uint32_t interface::_eosio_f64_trunc_i32u( double af ) const { + float64_t a = to_softfloat64(af); + if (_eosio_f64_ge(af, 4294967296.0) || _eosio_f64_le(af, -1.0)) + FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f64.convert_u/i32 overflow"); + if (is_nan(a)) + FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f64.convert_u/i32 unrepresentable"); + return f64_to_ui32( to_softfloat64(_eosio_f64_trunc( af )), 0, false ); + } + int64_t interface::_eosio_f32_trunc_i64s( float af ) const { + float32_t a = to_softfloat32(af); + if (_eosio_f32_ge(af, 9223372036854775808.0f) || _eosio_f32_lt(af, -9223372036854775808.0f)) + FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f32.convert_s/i64 overflow"); + if (is_nan(a)) + FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f32.convert_s/i64 unrepresentable"); + return f32_to_i64( to_softfloat32(_eosio_f32_trunc( af )), 0, false ); + } + int64_t interface::_eosio_f64_trunc_i64s( double af ) const { + float64_t a = to_softfloat64(af); + if (_eosio_f64_ge(af, 9223372036854775808.0) || _eosio_f64_lt(af, -9223372036854775808.0)) + FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f64.convert_s/i64 overflow"); + if (is_nan(a)) + FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f64.convert_s/i64 unrepresentable"); + + return f64_to_i64( to_softfloat64(_eosio_f64_trunc( af )), 0, false ); + } + uint64_t interface::_eosio_f32_trunc_i64u( float af ) const { + float32_t a = to_softfloat32(af); + if (_eosio_f32_ge(af, 18446744073709551616.0f) || _eosio_f32_le(af, -1.0f)) + FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f32.convert_u/i64 overflow"); + if (is_nan(a)) + FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f32.convert_u/i64 unrepresentable"); + return f32_to_ui64( to_softfloat32(_eosio_f32_trunc( af )), 0, false ); + } + uint64_t interface::_eosio_f64_trunc_i64u( double af ) const { + float64_t a = to_softfloat64(af); + if (_eosio_f64_ge(af, 18446744073709551616.0) || _eosio_f64_le(af, -1.0)) + FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f64.convert_u/i64 overflow"); + if (is_nan(a)) + FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, f64.convert_u/i64 unrepresentable"); + return f64_to_ui64( to_softfloat64(_eosio_f64_trunc( af )), 0, false ); + } + float interface::_eosio_i32_to_f32( int32_t a ) const { + return from_softfloat32(i32_to_f32( a )); + } + float interface::_eosio_i64_to_f32( int64_t a ) const { + return from_softfloat32(i64_to_f32( a )); + } + float interface::_eosio_ui32_to_f32( uint32_t a ) const { + return from_softfloat32(ui32_to_f32( a )); + } + float interface::_eosio_ui64_to_f32( uint64_t a ) const { + return from_softfloat32(ui64_to_f32( a )); + } + double interface::_eosio_i32_to_f64( int32_t a ) const { + return from_softfloat64(i32_to_f64( a )); + } + double interface::_eosio_i64_to_f64( int64_t a ) const { + return from_softfloat64(i64_to_f64( a )); + } + double interface::_eosio_ui32_to_f64( uint32_t a ) const { + return from_softfloat64(ui32_to_f64( a )); + } + double interface::_eosio_ui64_to_f64( uint64_t a ) const { + return from_softfloat64(ui64_to_f64( a )); + } +}}} // ns eosio::chain::webassembly diff --git a/libraries/chain/webassembly/system.cpp b/libraries/chain/webassembly/system.cpp new file mode 100644 index 00000000000..ae51ec190cc --- /dev/null +++ b/libraries/chain/webassembly/system.cpp @@ -0,0 +1,22 @@ +#include +#include +#include + +namespace eosio { namespace chain { namespace webassembly { + /* these are both unfortunate that we didn't make the return type an int64_t */ + uint64_t interface::current_time() const { + return static_cast( context.control.pending_block_time().time_since_epoch().count() ); + } + + uint64_t interface::publication_time() const { + return static_cast( context.trx_context.published.time_since_epoch().count() ); + } + + bool interface::is_feature_activated( legacy_ptr feature_digest ) const { + return context.control.is_protocol_feature_activated( *feature_digest ); + } + + name interface::get_sender() const { + return context.get_sender(); + } +}}} // ns eosio::chain::webassembly diff --git a/libraries/chain/webassembly/transaction.cpp b/libraries/chain/webassembly/transaction.cpp new file mode 100644 index 00000000000..3c7dbfacadb --- /dev/null +++ b/libraries/chain/webassembly/transaction.cpp @@ -0,0 +1,35 @@ +#include +#include +#include + +namespace eosio { namespace chain { namespace webassembly { + void interface::send_inline( legacy_span data ) { + //TODO: Why is this limit even needed? And why is it not consistently checked on actions in input or deferred transactions + EOS_ASSERT( data.size() < context.control.get_global_properties().configuration.max_inline_action_size, inline_action_too_big, + "inline action too big" ); + + action act; + fc::raw::unpack(data.data(), data.size(), act); + context.execute_inline(std::move(act)); + } + + void interface::send_context_free_inline( legacy_span data ) { + //TODO: Why is this limit even needed? And why is it not consistently checked on actions in input or deferred transactions + EOS_ASSERT( data.size() < context.control.get_global_properties().configuration.max_inline_action_size, inline_action_too_big, + "inline action too big" ); + + action act; + fc::raw::unpack(data.data(), data.size(), act); + context.execute_context_free_inline(std::move(act)); + } + + void interface::send_deferred( legacy_ptr sender_id, account_name payer, legacy_span data, uint32_t replace_existing) { + transaction trx; + fc::raw::unpack(data.data(), data.size(), trx); + context.schedule_deferred_transaction(*sender_id, payer, std::move(trx), replace_existing); + } + + bool interface::cancel_deferred( legacy_ptr val ) { + return context.cancel_deferred_transaction( *val ); + } +}}} // ns eosio::chain::webassembly diff --git a/libraries/chain/webassembly/wabt.cpp b/libraries/chain/webassembly/wabt.cpp deleted file mode 100644 index 853960d312b..00000000000 --- a/libraries/chain/webassembly/wabt.cpp +++ /dev/null @@ -1,111 +0,0 @@ -#include -#include -#include -#include - -//wabt includes -#include -#include -#include - -namespace eosio { namespace chain { namespace webassembly { namespace wabt_runtime { - -//yep 🤮 -static wabt_apply_instance_vars* static_wabt_vars; - -using namespace wabt; -using namespace wabt::interp; -namespace wasm_constraints = eosio::chain::wasm_constraints; - -class wabt_instantiated_module : public wasm_instantiated_module_interface { - public: - wabt_instantiated_module(std::unique_ptr e, std::vector initial_mem, interp::DefinedModule* mod) : - _env(move(e)), _instatiated_module(mod), _initial_memory(initial_mem), - _executor(_env.get(), nullptr, Thread::Options(64*1024, - wasm_constraints::maximum_call_depth+2)) - { - for(Index i = 0; i < _env->GetGlobalCount(); ++i) { - if(_env->GetGlobal(i)->mutable_ == false) - continue; - _initial_globals.emplace_back(_env->GetGlobal(i), _env->GetGlobal(i)->typed_value); - } - - if(_env->GetMemoryCount()) - _initial_memory_configuration = _env->GetMemory(0)->page_limits; - } - - void apply(apply_context& context) override { - //reset mutable globals - for(const auto& mg : _initial_globals) - mg.first->typed_value = mg.second; - - wabt_apply_instance_vars this_run_vars{nullptr, context}; - static_wabt_vars = &this_run_vars; - - //reset memory to inital size & copy back in initial data - if(_env->GetMemoryCount()) { - Memory* memory = this_run_vars.memory = _env->GetMemory(0); - memory->page_limits = _initial_memory_configuration; - memory->data.resize(_initial_memory_configuration.initial * WABT_PAGE_SIZE); - memcpy(memory->data.data(), _initial_memory.data(), _initial_memory.size()); - memset(memory->data.data() + _initial_memory.size(), 0, memory->data.size() - _initial_memory.size()); - } - - _params[0].set_i64(context.get_receiver().to_uint64_t()); - _params[1].set_i64(context.get_action().account.to_uint64_t()); - _params[2].set_i64(context.get_action().name.to_uint64_t()); - - ExecResult res = _executor.RunStartFunction(_instatiated_module); - EOS_ASSERT( res.result == interp::Result::Ok, wasm_execution_error, "wabt start function failure (${s})", ("s", ResultToString(res.result)) ); - - res = _executor.RunExportByName(_instatiated_module, "apply", _params); - EOS_ASSERT( res.result == interp::Result::Ok, wasm_execution_error, "wabt execution failure (${s})", ("s", ResultToString(res.result)) ); - } - - private: - std::unique_ptr _env; - DefinedModule* _instatiated_module; //this is owned by the Environment - std::vector _initial_memory; - TypedValues _params{3, TypedValue(Type::I64)}; - std::vector> _initial_globals; - Limits _initial_memory_configuration; - Executor _executor; -}; - -wabt_runtime::wabt_runtime() {} - -bool wabt_runtime::inject_module(IR::Module& module) { - wasm_injections::wasm_binary_injection injector(module); - injector.inject(); - return true; -} - -std::unique_ptr wabt_runtime::instantiate_module(const char* code_bytes, size_t code_size, std::vector initial_memory, - const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) { - std::unique_ptr env = std::make_unique(); - for(auto it = intrinsic_registrator::get_map().begin() ; it != intrinsic_registrator::get_map().end(); ++it) { - interp::HostModule* host_module = env->AppendHostModule(it->first); - for(auto itf = it->second.begin(); itf != it->second.end(); ++itf) { - host_module->AppendFuncExport(itf->first, itf->second.sig, [fn=itf->second.func](const auto* f, const auto* fs, const auto& args, auto& res) { - TypedValue ret = fn(*static_wabt_vars, args); - if(ret.type != Type::Void) - res[0] = ret; - return interp::Result::Ok; - }); - } - } - - interp::DefinedModule* instantiated_module = nullptr; - wabt::Errors errors; - - wabt::Result res = ReadBinaryInterp(env.get(), code_bytes, code_size, read_binary_options, &errors, &instantiated_module); - EOS_ASSERT( Succeeded(res), wasm_execution_error, "Error building wabt interp: ${e}", ("e", wabt::FormatErrorsToString(errors, Location::Type::Binary)) ); - - return std::make_unique(std::move(env), initial_memory, instantiated_module); -} - -void wabt_runtime::immediately_exit_currently_running_module() { - throw wasm_exit(); -} - -}}}} diff --git a/libraries/chain/whitelisted_intrinsics.cpp b/libraries/chain/whitelisted_intrinsics.cpp index cd3974d1ce5..021e93b19df 100644 --- a/libraries/chain/whitelisted_intrinsics.cpp +++ b/libraries/chain/whitelisted_intrinsics.cpp @@ -4,9 +4,9 @@ namespace eosio { namespace chain { template - bool find_intrinsic_helper( uint64_t h, const std::string& name, Iterator& itr, const Iterator& end ) { + bool find_intrinsic_helper( uint64_t h, std::string_view name, Iterator& itr, const Iterator& end ) { for( ; itr != end && itr->first == h; ++itr ) { - if( itr->second.compare( 0, itr->second.size(), name.c_str(), name.size() ) == 0 ) { + if( itr->second.compare( 0, itr->second.size(), name.data(), name.size() ) == 0 ) { return true; } } @@ -15,7 +15,7 @@ namespace eosio { namespace chain { } whitelisted_intrinsics_type::iterator - find_intrinsic( whitelisted_intrinsics_type& whitelisted_intrinsics, uint64_t h, const std::string& name ) + find_intrinsic( whitelisted_intrinsics_type& whitelisted_intrinsics, uint64_t h, std::string_view name ) { auto itr = whitelisted_intrinsics.lower_bound( h ); const auto end = whitelisted_intrinsics.end(); @@ -27,7 +27,7 @@ namespace eosio { namespace chain { } whitelisted_intrinsics_type::const_iterator - find_intrinsic( const whitelisted_intrinsics_type& whitelisted_intrinsics, uint64_t h, const std::string& name ) + find_intrinsic( const whitelisted_intrinsics_type& whitelisted_intrinsics, uint64_t h, std::string_view name ) { auto itr = whitelisted_intrinsics.lower_bound( h ); const auto end = whitelisted_intrinsics.end(); @@ -38,9 +38,9 @@ namespace eosio { namespace chain { return itr; } - bool is_intrinsic_whitelisted( const whitelisted_intrinsics_type& whitelisted_intrinsics, const std::string& name ) + bool is_intrinsic_whitelisted( const whitelisted_intrinsics_type& whitelisted_intrinsics, std::string_view name ) { - uint64_t h = static_cast( std::hash{}( name ) ); + uint64_t h = static_cast( std::hash{}( name ) ); auto itr = whitelisted_intrinsics.lower_bound( h ); const auto end = whitelisted_intrinsics.end(); @@ -48,29 +48,29 @@ namespace eosio { namespace chain { } - void add_intrinsic_to_whitelist( whitelisted_intrinsics_type& whitelisted_intrinsics, const std::string& name ) + void add_intrinsic_to_whitelist( whitelisted_intrinsics_type& whitelisted_intrinsics, std::string_view name ) { - uint64_t h = static_cast( std::hash{}( name ) ); + uint64_t h = static_cast( std::hash{}( name ) ); auto itr = find_intrinsic( whitelisted_intrinsics, h, name ); EOS_ASSERT( itr == whitelisted_intrinsics.end(), database_exception, "cannot add intrinsic '${name}' since it already exists in the whitelist", - ("name", name) + ("name", std::string(name)) ); whitelisted_intrinsics.emplace( std::piecewise_construct, std::forward_as_tuple( h ), - std::forward_as_tuple( name.c_str(), name.size(), + std::forward_as_tuple( name.data(), name.size(), whitelisted_intrinsics.get_allocator() ) ); } - void remove_intrinsic_from_whitelist( whitelisted_intrinsics_type& whitelisted_intrinsics, const std::string& name ) + void remove_intrinsic_from_whitelist( whitelisted_intrinsics_type& whitelisted_intrinsics, std::string_view name ) { - uint64_t h = static_cast( std::hash{}( name ) ); + uint64_t h = static_cast( std::hash{}( name ) ); auto itr = find_intrinsic( whitelisted_intrinsics, h, name ); EOS_ASSERT( itr != whitelisted_intrinsics.end(), database_exception, "cannot remove intrinsic '${name}' since it does not exist in the whitelist", - ("name", name) + ("name", std::string(name)) ); whitelisted_intrinsics.erase( itr ); @@ -82,10 +82,10 @@ namespace eosio { namespace chain { whitelisted_intrinsics.clear(); for( const auto& name : s ) { - uint64_t h = static_cast( std::hash{}( name ) ); + uint64_t h = static_cast( std::hash{}( name ) ); whitelisted_intrinsics.emplace( std::piecewise_construct, std::forward_as_tuple( h ), - std::forward_as_tuple( name.c_str(), name.size(), + std::forward_as_tuple( name.data(), name.size(), whitelisted_intrinsics.get_allocator() ) ); } @@ -95,7 +95,7 @@ namespace eosio { namespace chain { std::set s; for( const auto& p : whitelisted_intrinsics ) { - s.emplace( p.second.c_str(), p.second.size() ); + s.emplace( p.second.data(), p.second.size() ); } return s; diff --git a/libraries/chain_kv/.clang-format b/libraries/chain_kv/.clang-format new file mode 100644 index 00000000000..b55bfd7924e --- /dev/null +++ b/libraries/chain_kv/.clang-format @@ -0,0 +1,76 @@ +BasedOnStyle: LLVM +IndentWidth: 3 +UseTab: Never +ColumnLimit: 120 + +--- +Language: Cpp +# always align * and & to the type +DerivePointerAlignment: false +PointerAlignment: Left + +# regroup includes to these classes +IncludeCategories: + - Regex: '(<|"(eosio)/)' + Priority: 4 + - Regex: '(<|"(boost)/)' + Priority: 3 + - Regex: '(<|"(llvm|llvm-c|clang|clang-c)/' + Priority: 3 + - Regex: '<[[:alnum:]]+>' + Priority: 2 + - Regex: '.*' + Priority: 1 + +#IncludeBlocks: Regroup + +# set indent for public, private and protected +#AccessModifierOffset: 3 + +# make line continuations twice the normal indent +ContinuationIndentWidth: 6 + +# add missing namespace comments +FixNamespaceComments: true + +# add spaces to braced list i.e. int* foo = { 0, 1, 2 }; instead of int* foo = {0,1,2}; +Cpp11BracedListStyle: false +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: true +AlignConsecutiveDeclarations: true +AlignOperands: true +AlignTrailingComments: true +AllowShortCaseLabelsOnASingleLine: true +AllowShortFunctionsOnASingleLine: All +AllowShortBlocksOnASingleLine: true +#AllowShortIfStatementsOnASingleLine: WithoutElse +#AllowShortIfStatementsOnASingleLine: true +#AllowShortLambdasOnASingleLine: All +AllowShortLoopsOnASingleLine: true +AlwaysBreakTemplateDeclarations: true + +BinPackParameters: true +### use this with clang9 +BreakBeforeBraces: Custom +BraceWrapping: + #AfterCaseLabel: true + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + +BreakConstructorInitializers: BeforeColon +CompactNamespaces: true +IndentCaseLabels: true +IndentPPDirectives: AfterHash +NamespaceIndentation: Inner +ReflowComments: true +SortIncludes: true +SortUsingDeclarations: true +--- diff --git a/libraries/chain_kv/CMakeLists.txt b/libraries/chain_kv/CMakeLists.txt new file mode 100644 index 00000000000..7a2b351be1b --- /dev/null +++ b/libraries/chain_kv/CMakeLists.txt @@ -0,0 +1,13 @@ +file(GLOB_RECURSE HEADERS "include/*.hpp") + +add_library( chain_kv INTERFACE ) + +target_link_libraries( chain_kv + INTERFACE fc rocksdb softfloat + ) + +target_include_directories( chain_kv + INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/../rocksdb/include" + ) + +add_subdirectory(unit_tests) diff --git a/libraries/chain_kv/README.md b/libraries/chain_kv/README.md new file mode 100644 index 00000000000..c88f5e8d3a3 --- /dev/null +++ b/libraries/chain_kv/README.md @@ -0,0 +1,90 @@ +# Chain-KV aka Session API + +### Table of contents +1. [Enable ChainKV in Nodeos](#enable-chainkv-in-nodeos) +2. [ChainKV or Chainbase](#chainkv-or-chainbase) +3. [Using a database table in a smart contract](#using-a-database-table-in-a-smart-contract) +4. [Using a kv table in a smart contract](#using-a-kv-table-in-a-smart-contract) +5. [What is a Session](#what-is-a-session) +6. [Extending Session](#extending-session) +7. [What is an Undo stack](#what-is-an-undo-stack) + +## Enable ChainKV in Nodeos + +To enable the use of the RocksDB backing store in nodeos provide this argument when starting nodeos `--backing-store rocksdb`. +By default Chainbase is the backing store of Nodeos. So when this argument is not present when starting Nodeos, Chainbase will be used. You can also +be explicit in setting the use of Chainbase by specifying `--backing-store chainbase` when starting Nodeos. + +## ChainKV or Chainbase + + +## Using a database table in a smart contract + + +## Using a kv table in a smart contract + + +## What is a Session + +A Session is a lexicographically ordered (by key) in memory key-value datastore. It provides methods for reading, writing and deleting key-value pairs and for ordered iteration over the keys. Sessions can be laid out in a parent-child relationship (in the fashion of a linked list) in such a way that you can form the building blocks of a block chain, where the head block would be the permanent, irrevisible data store (such as RocksDB). + +Session was implemented with the assumption that once a Session instance has a child, that Session instance has become immutable and the child Session instance will contain delta changes that can either be committed up into the parent Session instance or discarded (undo) by either breaking the parent child relationship or by calling undo on the child session. This abstraction allows for defining Session as either a block or a transaction within that block depending on the context. + +The Session type is templated to allow template specialization for introducing new permament data stores into the EOS system. Currently there are only two types of Sessions implemented, a RocksDB specialization to allow for persisting key value pairs in RocksDB and the default template which represents the in memory key-value datastore. + +## Extending Session + +The Session API depends mainly on duck typing and template specialization, so extending the Session API to introduce new data stores is a fairly straight forward process. For example if you wanted to introduce a new permament data store then the basic setup would be something like the following code example. For additional documentation refer to session.hpp. + +```cpp +/// This could be any type of datastore that you'd want to introduce. +/// For example any relational database engine, a http end point that streams data to another server, etc... +struct my_datastore_t {}; + +/// To extend Session, we just provide a specialization tagged by the struct above. +template <> +class session { + /// This API relies on duck typing, so this specialization needs to provide at least the same + /// public interface as the templated session defininition. Beyond that, you can provide + /// any additional methods you want to implement your specialization. Refer to session.hpp + /// for documentation on these methods. + + std::unordered_map deltas() const; + void attach(Parent& parent); + void attach(session& parent); + void detach(); + void undo(); + void commit(); + + std::optional read(const shared_bytes& key) const; + void write(const shared_bytes& key, const shared_bytes& value); + bool contains(const shared_bytes& key) const; + void erase(const shared_bytes& key); + void clear(); + + template + const std::pair>, std::unordered_set> + read(const Iterable& keys) const; + + template + void write(const Iterable& key_values); + + template + void erase(const Iterable& keys); + + template + void write_to(Other_data_store& ds, const Iterable& keys) const; + + template + void read_from(const Other_data_store& ds, const Iterable& keys); + + iterator find(const shared_bytes& key) const; + iterator begin() const; + iterator end() const; + iterator lower_bound(const shared_bytes& key) const; +}; +``` + +### What is an Undo stack + +The undo stack is a container used within the context of the EOS blockchain for managing sessions. While this is termed a `stack`, the way this container works is closer to an `std::deque`. The operations that are valid from the bottom of the stack would only be making the session irrevisible by committing it into the permament datastore. The exception to this would be when the bottom and the top of the stack are the same in which case you can also perform the operations valid on the top of stack. The bottom of the stack is normally referred to as the `root` session. The operations that are valid from the top of the stack include squashing (combining the top 2 session on the stack), popping the top session which discards the changes in that session (an undo operation) and pushing a new session instance onto the stack. The top of the stack is normally referred to as the `head` session. \ No newline at end of file diff --git a/libraries/chain_kv/include/b1/chain_kv/chain_kv.hpp b/libraries/chain_kv/include/b1/chain_kv/chain_kv.hpp new file mode 100644 index 00000000000..a3e0b8cfaa6 --- /dev/null +++ b/libraries/chain_kv/include/b1/chain_kv/chain_kv.hpp @@ -0,0 +1,1056 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace b1::chain_kv { + +class exception : public std::exception { + std::string msg; + + public: + exception(std::string&& msg) : msg(std::move(msg)) {} + exception(const exception&) = default; + exception(exception&&) = default; + + exception& operator=(const exception&) = default; + exception& operator=(exception&&) = default; + + const char* what() const noexcept override { return msg.c_str(); } +}; + +inline void check(rocksdb::Status s, const char* prefix) { + if (!s.ok()) + throw exception(prefix + s.ToString()); +} + +using bytes = std::vector; + +inline rocksdb::Slice to_slice(const bytes& v) { return { v.data(), v.size() }; } + +inline bytes to_bytes(const rocksdb::Slice& v) { return { v.data(), v.data() + v.size() }; } + +inline std::shared_ptr to_shared_bytes(const rocksdb::Slice& v) { + return std::make_shared(v.data(), v.data() + v.size()); +} + +// Bypasses fc's vector size limit +template +void pack_bytes(Stream& s, const bytes& b) { + fc::unsigned_int size(b.size()); + if (size.value != b.size()) + throw exception("bytes is too big"); + fc::raw::pack(s, size); + s.write(b.data(), b.size()); +} + +template +void pack_optional_bytes(Stream& s, const bytes* b) { + fc::raw::pack(s, bool(b)); + if (b) + pack_bytes(s, *b); +} + +template +std::pair get_bytes(Stream& s) { + fc::unsigned_int size; + fc::raw::unpack(s, size); + if (size > s.remaining()) + throw exception("bad size for bytes"); + auto data = s.pos(); + s.skip(size.value); + return { data, size.value }; +} + +template +std::pair get_optional_bytes(Stream& s) { + bool present; + fc::raw::unpack(s, present); + if (present) + return get_bytes(s); + else + return { nullptr, 0 }; +} + +template +inline int compare_blob(const A& a, const B& b) { + static_assert(std::is_same_v, char> || + std::is_same_v, unsigned char>); + static_assert(std::is_same_v, char> || + std::is_same_v, unsigned char>); + auto r = memcmp(a.data(), b.data(), std::min(a.size(), b.size())); + if (r) + return r; + if (a.size() < b.size()) + return -1; + if (a.size() > b.size()) + return 1; + return 0; +} + +struct less_blob { + using is_transparent = void; + + template + bool operator()(const A& a, const B& b) const { + return compare_blob(a, b) < 0; + } +}; + +inline bytes get_next_prefix(const bytes& prefix) { + bytes next_prefix = prefix; + while (!next_prefix.empty()) { + if (++next_prefix.back()) + break; + next_prefix.pop_back(); + } + return next_prefix; +} + +namespace detail { + using uint128_t = __uint128_t; + + template + struct value_storage; + + template + struct value_storage { + using type = T; + constexpr T* as_ptr(T* value) const { return (value); } + }; + + template + struct value_storage { + using type = T; + constexpr T* as_ptr(T& value) const { return &value; } + }; + + template + auto append_key(bytes& dest, T value) { + + using t_type = typename value_storage::type; + t_type t_array[N]; + const t_type* first = value_storage().as_ptr(value); + const t_type* last = first + N; + std::reverse_copy(first, last, std::begin(t_array)); + + char* t_array_as_char_begin = reinterpret_cast(t_array); + char* t_array_as_char_end = reinterpret_cast(t_array + N); + std::reverse(t_array_as_char_begin, t_array_as_char_end); + dest.insert(dest.end(), t_array_as_char_begin, t_array_as_char_end); + } + + template + auto insert_key(eosio::session::shared_bytes& dest, size_t index, T value) { + + using t_type = typename value_storage::type; + t_type t_array[N]; + const t_type* first = value_storage().as_ptr(value); + const t_type* last = first + N; + std::reverse_copy(first, last, std::begin(t_array)); + + char* t_array_as_char_begin = reinterpret_cast(t_array); + char* t_array_as_char_end = reinterpret_cast(t_array + N); + std::reverse(t_array_as_char_begin, t_array_as_char_end); + std::memcpy(dest.data() + index, t_array_as_char_begin, t_array_as_char_end - t_array_as_char_begin); + } + + template + UInt float_to_key(T value) { + static_assert(sizeof(T) == sizeof(UInt), "Expected unsigned int of the same size"); + UInt result; + std::memcpy(&result, &value, sizeof(T)); + const UInt signbit = (static_cast(1) << (std::numeric_limits::digits - 1)); + UInt mask = 0; + if (result == signbit) + result = 0; + if (result & signbit) + mask = ~mask; + return result ^ (mask | signbit); + } + + template + T key_to_float(UInt value) { + static_assert(sizeof(T) == sizeof(UInt), "Expected unsigned int of the same size"); + // encoded signbit indicates positive value + const UInt signbit = (static_cast(1) << (std::numeric_limits::digits - 1)); + UInt mask = 0; + if ((value & signbit) == 0) + mask = ~mask; + value = {value ^ (mask | signbit)}; + T float_result; + std::memcpy(&float_result, &value, sizeof(UInt)); + return float_result; + } + + template + bool extract_key(It& key_loc, It key_end, Key& key) { + const std::size_t distance = std::distance(key_loc, key_end); + using t_type = typename value_storage::type; + constexpr static auto key_size = sizeof(t_type) * N; + if (distance < key_size) + return false; + + key_end = key_loc + key_size; + t_type t_array[N]; + char* t_array_as_char_begin = reinterpret_cast(t_array); + std::copy(key_loc, key_loc + key_size, t_array_as_char_begin); + key_loc = key_end; + char* t_array_as_char_end = reinterpret_cast(t_array + N); + std::reverse(t_array_as_char_begin, t_array_as_char_end); + + t_type* key_ptr = value_storage().as_ptr(key); + std::reverse_copy(std::begin(t_array), std::end(t_array), key_ptr); + + return true; + } +} + +template +auto append_key(bytes& dest, T value) -> std::enable_if_t, void> { + detail::append_key(dest, value); +} + +template +auto append_key(bytes& dest, std::array value) -> std::enable_if_t, void> { + detail::append_key(dest, value.data()); +} + +inline void append_key(bytes& dest, float64_t value) { + auto float_key = detail::float_to_key(value); + detail::append_key(dest, float_key); +} + +inline void append_key(bytes& dest, float128_t value) { + auto float_key = detail::float_to_key(value); + // underlying storage is implemented as uint64_t[2], but it is laid out like it is uint128_t + detail::append_key(dest, float_key); +} + +template +auto insert_key(eosio::session::shared_bytes& dest, size_t index, T value) -> std::enable_if_t, void> { + detail::insert_key(dest, index, value); +} + +template +auto insert_key(eosio::session::shared_bytes& dest, size_t index, std::array value) -> std::enable_if_t, void> { + detail::insert_key(dest, index, value.data()); +} + +inline void insert_key(eosio::session::shared_bytes& dest, size_t index, float64_t value) { + auto float_key = detail::float_to_key(value); + detail::insert_key(dest, index, float_key); +} + +inline void insert_key(eosio::session::shared_bytes& dest, size_t index, float128_t value) { + auto float_key = detail::float_to_key(value); + // underlying storage is implemented as uint64_t[2], but it is laid out like it is uint128_t + detail::insert_key(dest, index, float_key); +} + +template +auto extract_key(It& key_loc, It key_end, T& key) -> std::enable_if_t, bool> { + return detail::extract_key(key_loc, key_end, key); +} + +template +auto extract_key(It& key_loc, It key_end, std::array& key) -> std::enable_if_t, bool> { + T* key_ptr = key.data(); + return detail::extract_key(key_loc, key_end, key_ptr); +} + +template +bool extract_key(It& key_loc, It key_end, float64_t& key) { + uint64_t int_key; + const bool extract = detail::extract_key(key_loc, key_end, int_key); + if (!extract) + return false; + + key = detail::key_to_float(int_key); + return true; +} + +template +bool extract_key(It& key_loc, It key_end, float128_t& key) { + detail::uint128_t int_key; + const bool extract = detail::extract_key(key_loc, key_end, int_key); + if (!extract) + return false; + + key = detail::key_to_float(int_key); + return true; +} + +template +bytes create_full_key(const bytes& prefix, uint64_t contract, const T& key) { + bytes result; + result.reserve(prefix.size() + sizeof(contract) + key.size()); + result.insert(result.end(), prefix.begin(), prefix.end()); + append_key(result, contract); + result.insert(result.end(), key.data(), key.data() + key.size()); + return result; +} + +struct database { + std::unique_ptr rdb; + + database(const char* db_path, bool create_if_missing, std::optional threads = {}, + std::optional max_open_files = {}) { + + rocksdb::Options options; + options.create_if_missing = create_if_missing; + options.level_compaction_dynamic_level_bytes = true; + options.bytes_per_sync = 1048576; + + if (threads) + options.IncreaseParallelism(*threads); + + options.OptimizeLevelStyleCompaction(256ull << 20); + + if (max_open_files) + options.max_open_files = *max_open_files; + + rocksdb::BlockBasedTableOptions table_options; + table_options.format_version = 4; + table_options.index_block_restart_interval = 16; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + rocksdb::DB* p; + check(rocksdb::DB::Open(options, db_path, &p), "database::database: rocksdb::DB::Open: "); + rdb.reset(p); + + // Sentinels with keys 0x00 and 0xff simplify iteration logic. + // Views have prefixes which must start with a byte within the range 0x01 - 0xfe. + rocksdb::WriteBatch batch; + bool modified = false; + auto write_sentinal = [&](const bytes& k) { + rocksdb::PinnableSlice v; + auto stat = rdb->Get(rocksdb::ReadOptions(), rdb->DefaultColumnFamily(), to_slice(k), &v); + if (stat.IsNotFound()) { + check(batch.Put(to_slice(k), {}), "database::database: rocksdb::WriteBatch::Put: "); + modified = true; + } else { + check(stat, "database::database: rocksdb::DB::Get: "); + } + }; + write_sentinal({ 0x00 }); + write_sentinal({ (char)0xff }); + if (modified) + write(batch); + } + + database(database&&) = default; + database& operator=(database&&) = default; + + void flush(bool allow_write_stall, bool wait) { + rocksdb::FlushOptions op; + op.allow_write_stall = allow_write_stall; + op.wait = wait; + rdb->Flush(op); + } + + void write(rocksdb::WriteBatch& batch) { + rocksdb::WriteOptions opt; + opt.disableWAL = true; + check(rdb->Write(opt, &batch), "database::write: rocksdb::DB::Write (batch)"); + batch.Clear(); + } +}; // database + +struct key_value { + rocksdb::Slice key = {}; + rocksdb::Slice value = {}; +}; + +inline int compare_key(const std::optional& a, const std::optional& b) { + // nullopt represents end; everything else is before end + if (!a && !b) + return 0; + else if (!a && b) + return 1; + else if (a && !b) + return -1; + else + return compare_blob(a->key, b->key); +} + +inline int compare_value(const std::shared_ptr& a, const std::shared_ptr& b) { + // nullptr represents erased; everything else, including empty, is after erased + if (!a && !b) + return 0; + else if (!a && b) + return -1; + else if (a && !b) + return 1; + else + return compare_blob(*a, *b); +} + +using cache_map = std::map; + +// The cache serves these needs: +// * Keep track of changes that need to be written to rocksdb +// * Support reading writes +// * Support iteration logic +struct cached_value { + uint64_t num_erases = 0; // For iterator invalidation + std::shared_ptr orig_value = {}; + std::shared_ptr current_value = {}; + bool in_change_list = false; + cache_map::iterator change_list_next = {}; +}; + +struct undo_state { + uint8_t format_version = 0; + int64_t revision = 0; + std::vector undo_stack = {}; // Number of undo segments needed to go back each revision + uint64_t next_undo_segment = 0; +}; + +template +void pack_undo_segment(Stream& s, const bytes& key, const bytes* old_value, const bytes* new_value) { + pack_bytes(s, key); + pack_optional_bytes(s, old_value); + pack_optional_bytes(s, new_value); +} + +class undo_stack { + private: + database& db; + bytes undo_prefix; + uint64_t target_segment_size; + bytes state_prefix; + bytes segment_prefix; + bytes segment_next_prefix; + undo_state state; + + public: + undo_stack(database& db, const bytes& undo_prefix, uint64_t target_segment_size = 64 * 1024 * 1024) + : db{ db }, undo_prefix{ undo_prefix }, target_segment_size{ target_segment_size } { + if (this->undo_prefix.empty()) + throw exception("undo_prefix is empty"); + + // Sentinels reserve 0x00 and 0xff. This keeps rocksdb iterators from going + // invalid during iteration. + if (this->undo_prefix[0] == 0x00 || this->undo_prefix[0] == (char)0xff) + throw exception("undo_stack may not have a prefix which begins with 0x00 or 0xff"); + + state_prefix = this->undo_prefix; + state_prefix.push_back(0x00); + segment_prefix = this->undo_prefix; + segment_prefix.push_back(0x80); + segment_next_prefix = get_next_prefix(segment_prefix); + + rocksdb::PinnableSlice v; + auto stat = db.rdb->Get(rocksdb::ReadOptions(), db.rdb->DefaultColumnFamily(), to_slice(this->state_prefix), &v); + if (!stat.IsNotFound()) + check(stat, "undo_stack::undo_stack: rocksdb::DB::Get: "); + if (stat.ok()) { + auto format_version = fc::raw::unpack(v.data(), v.size()); + if (format_version) + throw exception("invalid undo format"); + state = fc::raw::unpack(v.data(), v.size()); + } + } + + int64_t revision() const { return state.revision; } + int64_t first_revision() const { return state.revision - state.undo_stack.size(); } + + void set_revision(uint64_t revision, bool write_now = true) { + if (state.undo_stack.size() != 0) + throw exception("cannot set revision while there is an existing undo stack"); + if (revision > std::numeric_limits::max()) + throw exception("revision to set is too high"); + if (static_cast(revision) < state.revision) + throw exception("revision cannot decrease"); + state.revision = revision; + if (write_now) + write_state(); + } + + // Create a new entry on the undo stack + void push(bool write_now = true) { + state.undo_stack.push_back(0); + ++state.revision; + if (write_now) + write_state(); + } + + // Combine the top two states on the undo stack + void squash(bool write_now = true) { + if (state.undo_stack.empty()) { + return; + } else if (state.undo_stack.size() == 1) { + rocksdb::WriteBatch batch; + check(batch.DeleteRange(to_slice(create_segment_key(0)), + to_slice(create_segment_key(state.next_undo_segment))), + "undo_stack::squash: rocksdb::WriteBatch::DeleteRange: "); + state.undo_stack.clear(); + --state.revision; + write_state(batch); + db.write(batch); + return; + } + auto n = state.undo_stack.back(); + state.undo_stack.pop_back(); + state.undo_stack.back() += n; + --state.revision; + if (write_now) + write_state(); + } + + // Reset the contents to the state at the top of the undo stack + void undo(bool write_now = true) { + if (state.undo_stack.empty()) + throw exception("nothing to undo"); + rocksdb::WriteBatch batch; + + std::unique_ptr rocks_it{ db.rdb->NewIterator(rocksdb::ReadOptions()) }; + auto first = create_segment_key(state.next_undo_segment - state.undo_stack.back()); + rocks_it->Seek(to_slice(segment_next_prefix)); + if (rocks_it->Valid()) + rocks_it->Prev(); + + while (rocks_it->Valid()) { + auto segment_key = rocks_it->key(); + if (compare_blob(segment_key, first) < 0) + break; + write_now = true; + auto segment = rocks_it->value(); + fc::datastream ds(segment.data(), segment.size()); + while (ds.remaining()) { + auto [key, key_size] = get_bytes(ds); + auto [old_value, old_value_size] = get_optional_bytes(ds); + get_optional_bytes(ds); + if (old_value) + check(batch.Put({ key, key_size }, { old_value, old_value_size }), + "undo_stack::undo: rocksdb::WriteBatch::Put: "); + else + check(batch.Delete({ key, key_size }), "undo_stack::undo: rocksdb::WriteBatch::Delete: "); + } + check(batch.Delete(segment_key), "undo_stack::undo: rocksdb::WriteBatch::Delete: "); + rocks_it->Prev(); + } + check(rocks_it->status(), "undo_stack::undo: iterate rocksdb: "); + + state.next_undo_segment -= state.undo_stack.back(); + state.undo_stack.pop_back(); + --state.revision; + if (write_now) { + write_state(batch); + db.write(batch); + } + } + + // Discard all undo history prior to revision + void commit(int64_t revision) { + revision = std::min(revision, state.revision); + int64_t first_revision = state.revision - state.undo_stack.size(); + if (first_revision < revision) { + rocksdb::WriteBatch batch; + state.undo_stack.erase(state.undo_stack.begin(), state.undo_stack.begin() + (revision - first_revision)); + uint64_t keep_undo_segment = state.next_undo_segment; + for (auto n : state.undo_stack) // + keep_undo_segment -= n; + check(batch.DeleteRange(to_slice(create_segment_key(0)), to_slice(create_segment_key(keep_undo_segment))), + "undo_stack::commit: rocksdb::WriteBatch::DeleteRange: "); + write_state(batch); + db.write(batch); + } + } + + // Write changes in `change_list`. Everything in `change_list` must belong to `cache`. + // + // It is undefined behavior if any `orig_value` in `change_list` doesn't match the + // database's current state. + void write_changes(cache_map& cache, cache_map::iterator change_list) { + rocksdb::WriteBatch batch; + bytes segment; + segment.reserve(target_segment_size); + + auto write_segment = [&] { + if (segment.empty()) + return; + auto key = create_segment_key(state.next_undo_segment++); + check(batch.Put(to_slice(key), to_slice(segment)), "undo_stack::write_changes: rocksdb::WriteBatch::Put: "); + ++state.undo_stack.back(); + segment.clear(); + }; + + auto append_segment = [&](auto f) { + fc::datastream size_stream; + f(size_stream); + if (segment.size() + size_stream.tellp() > target_segment_size) + write_segment(); + auto orig_size = segment.size(); + segment.resize(segment.size() + size_stream.tellp()); + fc::datastream ds(segment.data() + orig_size, size_stream.tellp()); + f(ds); + }; + + auto it = change_list; + while (it != cache.end()) { + if (compare_value(it->second.orig_value, it->second.current_value)) { + if (it->second.current_value) + check(batch.Put(to_slice(it->first), to_slice(*it->second.current_value)), + "undo_stack::write_changes: rocksdb::WriteBatch::Put: "); + else + check(batch.Delete(to_slice(it->first)), "undo_stack::write_changes: rocksdb::WriteBatch::Erase: "); + if (!state.undo_stack.empty()) { + append_segment([&](auto& stream) { + pack_undo_segment(stream, it->first, it->second.orig_value.get(), it->second.current_value.get()); + }); + } + } + it = it->second.change_list_next; + } + + write_segment(); + write_state(batch); + db.write(batch); + } // write_changes() + + void write_state() { + rocksdb::WriteBatch batch; + write_state(batch); + db.write(batch); + } + + private: + void write_state(rocksdb::WriteBatch& batch) { + check(batch.Put(to_slice(state_prefix), to_slice(fc::raw::pack(state))), + "undo_stack::write_state: rocksdb::WriteBatch::Put: "); + } + + bytes create_segment_key(uint64_t segment) { + bytes key; + key.reserve(segment_prefix.size() + sizeof(segment)); + key.insert(key.end(), segment_prefix.begin(), segment_prefix.end()); + append_key(key, segment); + return key; + } +}; // undo_stack + +// Supports reading and writing through a cache_map +// +// Caution: write_session will misbehave if it's used to read or write a key that +// that is changed elsewhere during write_session's lifetime. e.g. through another +// simultaneous write_session, unless that write_session is never written to the +// database. +// +// Extra keys stored in the cache used only as sentinels are exempt from this +// restriction. +struct write_session { + database& db; + const rocksdb::Snapshot* snapshot; + cache_map cache; + cache_map::iterator change_list = cache.end(); + + write_session(database& db, const rocksdb::Snapshot* snapshot = nullptr) : db{ db }, snapshot{ snapshot } {} + + rocksdb::ReadOptions read_options() { + rocksdb::ReadOptions r; + r.snapshot = snapshot; + return r; + } + + // Add item to change_list + void changed(cache_map::iterator it) { + if (it->second.in_change_list) + return; + it->second.in_change_list = true; + it->second.change_list_next = change_list; + change_list = it; + } + + // Get a value. Includes any changes written to cache. Returns nullptr + // if key-value doesn't exist. + std::shared_ptr get(bytes&& k) { + auto it = cache.find(k); + if (it != cache.end()) + return it->second.current_value; + + rocksdb::PinnableSlice v; + auto stat = db.rdb->Get(read_options(), db.rdb->DefaultColumnFamily(), to_slice(k), &v); + if (stat.IsNotFound()) + return nullptr; + check(stat, "write_session::get: rocksdb::DB::Get: "); + + auto value = to_shared_bytes(v); + cache[std::move(k)] = cached_value{ 0, value, value }; + return value; + } + + // Write a key-value to cache and add to change_list if changed. + void set(bytes&& k, const rocksdb::Slice& v) { + auto it = cache.find(k); + if (it != cache.end()) { + if (!it->second.current_value || compare_blob(*it->second.current_value, v)) { + it->second.current_value = to_shared_bytes(v); + changed(it); + } + return; + } + + rocksdb::PinnableSlice orig_v; + auto stat = db.rdb->Get(read_options(), db.rdb->DefaultColumnFamily(), to_slice(k), &orig_v); + if (stat.IsNotFound()) { + auto [it, b] = + cache.insert(cache_map::value_type{ std::move(k), cached_value{ 0, nullptr, to_shared_bytes(v) } }); + changed(it); + return; + } + + check(stat, "write_session::set: rocksdb::DB::Get: "); + if (compare_blob(v, orig_v)) { + auto [it, b] = cache.insert( + cache_map::value_type{ std::move(k), cached_value{ 0, to_shared_bytes(orig_v), to_shared_bytes(v) } }); + changed(it); + } else { + auto value = to_shared_bytes(orig_v); + cache[std::move(k)] = cached_value{ 0, value, value }; + } + } + + // Mark key as erased in the cache and add to change_list if changed. Bumps `num_erases` to invalidate iterators. + void erase(bytes&& k) { + { + auto it = cache.find(k); + if (it != cache.end()) { + if (it->second.current_value) { + ++it->second.num_erases; + it->second.current_value = nullptr; + changed(it); + } + return; + } + } + + rocksdb::PinnableSlice orig_v; + auto stat = db.rdb->Get(read_options(), db.rdb->DefaultColumnFamily(), to_slice(k), &orig_v); + if (stat.IsNotFound()) { + cache[std::move(k)] = cached_value{ 0, nullptr, nullptr }; + return; + } + + check(stat, "write_session::erase: rocksdb::DB::Get: "); + auto [it, b] = + cache.insert(cache_map::value_type{ std::move(k), cached_value{ 1, to_shared_bytes(orig_v), nullptr } }); + changed(it); + } + + // Fill cache with a key-value pair read from the database. Does not undo any changes (e.g. set() or erase()) + // already made to the cache. Returns an iterator to the freshly-created or already-existing cache entry. + cache_map::iterator fill_cache(const rocksdb::Slice& k, const rocksdb::Slice& v) { + cache_map::iterator it; + bool b; + it = cache.find(k); + if (it != cache.end()) + return it; + auto value = to_shared_bytes(v); + std::tie(it, b) = cache.insert(cache_map::value_type{ to_bytes(k), cached_value{ 0, value, value } }); + return it; + } + + // Write changes in `change_list` to database. See undo_stack::write_changes. + // + // Caution: write_changes wipes the cache, which invalidates iterators + void write_changes(undo_stack& u) { + u.write_changes(cache, change_list); + wipe_cache(); + } + + // Wipe the cache. Invalidates iterators. + void wipe_cache() { + cache.clear(); + change_list = cache.end(); + } +}; // write_session + +// A view of the database with a restricted range (prefix). Implements part of +// https://github.com/EOSIO/spec-repo/blob/master/esr_key_value_database_intrinsics.md +// including iterator wrap-around behavior. +// +// Keys have this format: prefix, contract, user-provided key. +class view { + public: + class iterator; + + chain_kv::write_session& write_session; + const bytes prefix; + + private: + struct iterator_impl { + friend chain_kv::view; + friend iterator; + + chain_kv::view& view; + bytes prefix; + size_t hidden_prefix_size; + bytes next_prefix; + cache_map::iterator cache_it; + uint64_t cache_it_num_erases = 0; + std::unique_ptr rocks_it; + + iterator_impl(chain_kv::view& view, uint64_t contract, const rocksdb::Slice& prefix) + : view{ view }, // + prefix{ create_full_key(view.prefix, contract, prefix) }, // + hidden_prefix_size{ view.prefix.size() + sizeof(contract) }, // + rocks_it{ view.write_session.db.rdb->NewIterator(view.write_session.read_options()) } // + { + next_prefix = get_next_prefix(this->prefix); + + // Fill the cache with sentinel keys to simplify iteration logic. These may be either + // the reserved 0x00 or 0xff sentinels, or keys from regions neighboring prefix. + rocks_it->Seek(to_slice(this->prefix)); + check(rocks_it->status(), "view::iterator_impl::iterator_impl: rocksdb::Iterator::Seek: "); + view.write_session.fill_cache(rocks_it->key(), rocks_it->value()); + rocks_it->Prev(); + check(rocks_it->status(), "view::iterator_impl::iterator_impl: rocksdb::Iterator::Prev: "); + view.write_session.fill_cache(rocks_it->key(), rocks_it->value()); + rocks_it->Seek(to_slice(next_prefix)); + check(rocks_it->status(), "view::iterator_impl::iterator_impl: rocksdb::Iterator::Seek: "); + view.write_session.fill_cache(rocks_it->key(), rocks_it->value()); + + move_to_end(); + } + + iterator_impl(const iterator_impl&) = delete; + iterator_impl& operator=(const iterator_impl&) = delete; + + void move_to_begin() { lower_bound_full_key(prefix); } + + void move_to_end() { cache_it = view.write_session.cache.end(); } + + void lower_bound(const char* key, size_t size) { + auto x = compare_blob(rocksdb::Slice{ key, size }, rocksdb::Slice{ prefix.data() + hidden_prefix_size, + prefix.size() - hidden_prefix_size }); + if (x < 0) { + key = prefix.data() + hidden_prefix_size; + size = prefix.size() - hidden_prefix_size; + } + + bytes full_key; + full_key.reserve(hidden_prefix_size + size); + full_key.insert(full_key.end(), prefix.data(), prefix.data() + hidden_prefix_size); + full_key.insert(full_key.end(), key, key + size); + lower_bound_full_key(full_key); + } + + void lower_bound_full_key(const bytes& full_key) { + rocks_it->Seek(to_slice(full_key)); + check(rocks_it->status(), "view::iterator_impl::lower_bound_full_key: rocksdb::Iterator::Seek: "); + cache_it = view.write_session.fill_cache(rocks_it->key(), rocks_it->value()); + if (compare_blob(cache_it->first, to_slice(full_key))) + cache_it = view.write_session.cache.lower_bound(full_key); + while (!cache_it->second.current_value) { + while (compare_blob(rocks_it->key(), cache_it->first) <= 0) { + rocks_it->Next(); + check(rocks_it->status(), "view::iterator_impl::lower_bound_full_key: rocksdb::Iterator::Next: "); + view.write_session.fill_cache(rocks_it->key(), rocks_it->value()); + } + ++cache_it; + } + if (compare_blob(cache_it->first, next_prefix) >= 0) + cache_it = view.write_session.cache.end(); + else + cache_it_num_erases = cache_it->second.num_erases; + } + + std::optional get_kv() { + if (cache_it == view.write_session.cache.end()) + return {}; + if (cache_it_num_erases != cache_it->second.num_erases) + throw exception("kv iterator is at an erased value"); + return key_value{ rocksdb::Slice{ cache_it->first.data() + hidden_prefix_size, + cache_it->first.size() - hidden_prefix_size }, + to_slice(*cache_it->second.current_value) }; + } + + bool is_end() { return cache_it == view.write_session.cache.end(); } + + bool is_valid() { + return cache_it != view.write_session.cache.end() && cache_it_num_erases == cache_it->second.num_erases; + } + + bool is_erased() { + return cache_it != view.write_session.cache.end() && cache_it_num_erases != cache_it->second.num_erases; + } + + iterator_impl& operator++() { + if (cache_it == view.write_session.cache.end()) { + move_to_begin(); + return *this; + } else if (cache_it_num_erases != cache_it->second.num_erases) + throw exception("kv iterator is at an erased value"); + do { + while (compare_blob(rocks_it->key(), cache_it->first) <= 0) { + rocks_it->Next(); + check(rocks_it->status(), "view::iterator_impl::operator++: rocksdb::Iterator::Next: "); + view.write_session.fill_cache(rocks_it->key(), rocks_it->value()); + } + ++cache_it; + } while (!cache_it->second.current_value); + if (compare_blob(cache_it->first, next_prefix) >= 0) + cache_it = view.write_session.cache.end(); + else + cache_it_num_erases = cache_it->second.num_erases; + return *this; + } + + iterator_impl& operator--() { + if (cache_it == view.write_session.cache.end()) { + rocks_it->Seek(to_slice(next_prefix)); + check(rocks_it->status(), "view::iterator_impl::operator--: rocksdb::Iterator::Seek: "); + cache_it = view.write_session.fill_cache(rocks_it->key(), rocks_it->value()); + if (compare_blob(cache_it->first, to_slice(next_prefix))) + cache_it = view.write_session.cache.lower_bound(next_prefix); + } else if (cache_it_num_erases != cache_it->second.num_erases) + throw exception("kv iterator is at an erased value"); + do { + while (compare_blob(rocks_it->key(), cache_it->first) >= 0) { + rocks_it->Prev(); + check(rocks_it->status(), "view::iterator_impl::operator--: rocksdb::Iterator::Prev: "); + view.write_session.fill_cache(rocks_it->key(), rocks_it->value()); + } + --cache_it; + } while (!cache_it->second.current_value); + if (compare_blob(cache_it->first, prefix) < 0) + cache_it = view.write_session.cache.end(); + else + cache_it_num_erases = cache_it->second.num_erases; + return *this; + } + }; // iterator_impl + + public: + // Iterates through a user-provided prefix within view's keyspace. + class iterator { + friend view; + + private: + std::unique_ptr impl; + + void check_initialized() const { + if (!impl) + throw exception("kv iterator is not initialized"); + } + + public: + iterator(view& view, uint64_t contract, const rocksdb::Slice& prefix) + : impl{ std::make_unique(view, contract, std::move(prefix)) } {} + + iterator(const iterator&) = delete; + iterator(iterator&&) = default; + + iterator& operator=(const iterator&) = delete; + iterator& operator=(iterator&&) = default; + + // Compare 2 iterators. Throws if the iterators are from different views. + // Also throws if either iterator is at an erased kv pair. non-end iterators + // compare less than end iterators. + friend int compare(const iterator& a, const iterator& b) { + a.check_initialized(); + b.check_initialized(); + if (&a.impl->view != &b.impl->view) + throw exception("iterators are from different views"); + return compare_key(a.impl->get_kv(), b.impl->get_kv()); + } + + friend bool operator==(const iterator& a, const iterator& b) { return compare(a, b) == 0; } + friend bool operator!=(const iterator& a, const iterator& b) { return compare(a, b) != 0; } + friend bool operator<(const iterator& a, const iterator& b) { return compare(a, b) < 0; } + friend bool operator<=(const iterator& a, const iterator& b) { return compare(a, b) <= 0; } + friend bool operator>(const iterator& a, const iterator& b) { return compare(a, b) > 0; } + friend bool operator>=(const iterator& a, const iterator& b) { return compare(a, b) >= 0; } + + iterator& operator++() { + check_initialized(); + ++*impl; + return *this; + } + + iterator& operator--() { + check_initialized(); + --*impl; + return *this; + } + + void move_to_begin() { + check_initialized(); + impl->move_to_begin(); + } + + void move_to_end() { + check_initialized(); + impl->move_to_end(); + } + + void lower_bound(const char* key, size_t size) { + check_initialized(); + impl->lower_bound(key, size); + } + + void lower_bound(const bytes& key) { lower_bound(key.data(), key.size()); } + + bool is_end() const { + check_initialized(); + return impl->is_end(); + } + + // true if !is_end() and not at an erased kv pair + bool is_valid() const { + check_initialized(); + return impl->is_valid(); + } + + // true if !is_end() and is at an erased kv pair + bool is_erased() const { + check_initialized(); + return impl->is_erased(); + } + + // Get key_value at current position. Returns nullopt if at end. Throws if at an erased kv pair. + // The returned key does not include the view's prefix or the contract. + std::optional get_kv() const { + check_initialized(); + return impl->get_kv(); + } + }; + + view(struct write_session& write_session, bytes prefix) + : write_session{ write_session }, prefix{ std::move(prefix) } { + if (this->prefix.empty()) + throw exception("kv view may not have empty prefix"); + + // Sentinels reserve 0x00 and 0xff. This keeps rocksdb iterators from going + // invalid during iteration. This also allows get_next_prefix() to function correctly. + if (this->prefix[0] == 0x00 || this->prefix[0] == (char)0xff) + throw exception("view may not have a prefix which begins with 0x00 or 0xff"); + } + + // Get a value. Includes any changes written to cache. Returns nullptr + // if key doesn't exist. + std::shared_ptr get(uint64_t contract, const rocksdb::Slice& k) { + return write_session.get(create_full_key(prefix, contract, k)); + } + + // Set a key-value pair + void set(uint64_t contract, const rocksdb::Slice& k, const rocksdb::Slice& v) { + write_session.set(create_full_key(prefix, contract, k), v); + } + + // Erase a key-value pair + void erase(uint64_t contract, const rocksdb::Slice& k) { write_session.erase(create_full_key(prefix, contract, k)); } +}; // view + +} // namespace b1::chain_kv + +FC_REFLECT(b1::chain_kv::undo_state, (format_version)(revision)(undo_stack)(next_undo_segment)) diff --git a/libraries/chain_kv/include/b1/session/rocks_session.hpp b/libraries/chain_kv/include/b1/session/rocks_session.hpp new file mode 100644 index 00000000000..b7f3119d5c9 --- /dev/null +++ b/libraries/chain_kv/include/b1/session/rocks_session.hpp @@ -0,0 +1,566 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +namespace eosio::session { + +/// \brief A tag type used to create the RocksDB session specialization. +/// \remarks To instantiate a RocksDB session use the following syntax auto db = session{...}; +struct rocksdb_t {}; + +/// \brief A specialization of session that interacts with a RocksDB instance instead of an in-memory cache. +/// \remarks The interface on this session type should work just like the non specialized version of session. +/// For more documentation on methods in this header, refer to the session header file. +template <> +class session { + public: + template + friend class session; + + template + class rocks_iterator { + public: + friend session; + + using difference_type = typename Iterator_traits::difference_type; + using value_type = typename Iterator_traits::value_type; + using pointer = typename Iterator_traits::pointer; + using reference = typename Iterator_traits::reference; + using iterator_category = typename Iterator_traits::iterator_category; + + rocks_iterator() = default; + rocks_iterator(const rocks_iterator& other); + rocks_iterator(rocks_iterator&& other); + rocks_iterator(session& session, rocksdb::Iterator& rit, int64_t index = -1); + ~rocks_iterator(); + + rocks_iterator& operator=(const rocks_iterator& other); + rocks_iterator& operator=(rocks_iterator&& other); + + rocks_iterator& operator++(); + rocks_iterator& operator--(); + value_type operator*() const; + value_type operator->() const; + bool operator==(const rocks_iterator& other) const; + bool operator!=(const rocks_iterator& other) const; + + shared_bytes key() const; + bool deleted() const; + + private: + void reset(); + + private: + session* m_session{ nullptr }; + rocksdb::Iterator* m_iterator{ nullptr }; + int64_t m_index{ -1 }; + }; + + struct iterator_traits { + using difference_type = std::ptrdiff_t; + using value_type = std::pair>; + using pointer = value_type*; + using reference = value_type&; + using iterator_category = std::bidirectional_iterator_tag; + }; + using iterator = rocks_iterator; + + public: + session() = default; + session(const session&) = default; + session(session&&) = default; + + /// \brief Constructor + /// \param db A pointer to the RocksDB db type instance. + /// \param max_iterators This type will cache up to max_iterators RocksDB iterator instances. + session(std::shared_ptr db, size_t max_iterators); + + session& operator=(const session&) = default; + session& operator=(session&&) = default; + + std::unordered_set updated_keys() const; + std::unordered_set deleted_keys() const; + + std::optional read(const shared_bytes& key); + void write(const shared_bytes& key, const shared_bytes& value); + bool contains(const shared_bytes& key); + void erase(const shared_bytes& key); + void clear(); + bool is_deleted(const shared_bytes& key) const; + + template + const std::pair>, std::unordered_set> + read(const Iterable& keys); + + template + void write(const Iterable& key_values); + + template + void erase(const Iterable& keys); + + template + void write_to(Other_data_store& ds, const Iterable& keys); + + template + void read_from(Other_data_store& ds, const Iterable& keys); + + iterator find(const shared_bytes& key); + iterator begin(); + iterator end(); + iterator lower_bound(const shared_bytes& key); + + void undo(); + void commit(); + + /// \brief Forces a flush on the underlying RocksDB db instance. + void flush(); + + static void destroy(const std::string& db_name); + + /// \brief User specified write options that are applied when writing or erasing data from RocksDB. + rocksdb::WriteOptions& write_options(); + + /// \brief User specified write options that are applied when writing or erasing data from RocksDB. + const rocksdb::WriteOptions& write_options() const; + + /// \brief User specified read options that are applied when reading or searching data from RocksDB. + rocksdb::ReadOptions& read_options(); + + /// \brief User specified read options that are applied when reading or searching data from RocksDB. + const rocksdb::ReadOptions& read_options() const; + + /// \brief The column family associated with this instance of the RocksDB session. + std::shared_ptr& column_family(); + + /// \brief The column family associated with this instance of the RocksDB session. + std::shared_ptr column_family() const; + + protected: + template + const std::pair>, std::unordered_set> + read_(const Iterable& keys); + + /// \brief Prepares an iterator. + /// \tparam Predicate A functor used for positioning the RocksDB iterator. It has the given signature + /// void(rocksdb::Iterator&) + /// \param setup The functor instance. + /// \remarks This method will first try to acquire a cached rocks db iterator. If none are available, + /// then it will construct a new rocksdb iterator instance that is owned by the session iterator instance + /// and will be destroyed when the session iterator goes out of scope. In the case that a cached rocks + /// db iterator was acquired, that rocks db iterator will be released back to the cache when the + /// session iterator instance goes out of scope. + template + iterator make_iterator_(const Predicate& setup) const; + + /// \brief Returns the active column family of this session. + /// \remarks If there is no user defined column family, this method will return the RocksDB default column family. + rocksdb::ColumnFamilyHandle* column_family_() const; + + private: + std::shared_ptr m_db; + std::shared_ptr m_column_family; + rocksdb::ReadOptions m_read_options; + rocksdb::ReadOptions m_iterator_read_options; + rocksdb::WriteOptions m_write_options; + + /// \brief The cache of RocksDB iterators. + mutable std::vector> m_iterators; + + /// \brief A list of the available indices in the iterator cache that are available for use. + mutable std::vector m_free_list; +}; + +inline session make_session(std::shared_ptr db, size_t max_iterators) { + return { std::move(db), max_iterators }; +} + +inline session::session(std::shared_ptr db, size_t max_iterators) + : m_db{ [&]() { + EOS_ASSERT(db, eosio::chain::database_exception, "db parameter cannot be null"); + return std::move(db); + }() }, + m_iterator_read_options{ [&]() { + auto read_options = rocksdb::ReadOptions{}; + read_options.verify_checksums = false; + read_options.fill_cache = false; + read_options.background_purge_on_iterator_cleanup = true; + return read_options; + }() }, + m_iterators{ [&]() { + auto iterators = decltype(m_iterators){}; + iterators.reserve(max_iterators); + + auto column_family = column_family_(); + for (size_t i = 0; i < max_iterators; ++i) { + iterators.emplace_back(m_db->NewIterator(m_iterator_read_options, column_family)); + } + return iterators; + }() }, + m_free_list{ [&]() { + auto list = decltype(m_free_list)(m_iterators.size()); + for (size_t i = 0; i < m_iterators.size(); ++i) { list[i] = i; } + return list; + }() } { + m_write_options.disableWAL = true; +} + +inline std::unordered_set session::updated_keys() const { return {}; } + +inline std::unordered_set session::deleted_keys() const { return {}; } + +inline void session::undo() {} + +inline void session::commit() {} + +inline bool session::is_deleted(const shared_bytes& key) const { return false; } + +inline std::optional session::read(const shared_bytes& key) { + auto key_slice = rocksdb::Slice{ key.data(), key.size() }; + auto pinnable_value = rocksdb::PinnableSlice{}; + auto status = m_db->Get(m_read_options, column_family_(), key_slice, &pinnable_value); + + if (status.code() != rocksdb::Status::Code::kOk) { + return {}; + } + + return shared_bytes(pinnable_value.data(), pinnable_value.size()); +} + +inline void session::write(const shared_bytes& key, const shared_bytes& value) { + auto key_slice = rocksdb::Slice{ key.data(), key.size() }; + auto value_slice = rocksdb::Slice{ value.data(), value.size() }; + auto status = m_db->Put(m_write_options, column_family_(), key_slice, value_slice); +} + +inline bool session::contains(const shared_bytes& key) { + auto key_slice = rocksdb::Slice{ key.data(), key.size() }; + auto value = std::string{}; + return m_db->KeyMayExist(m_read_options, column_family_(), key_slice, &value); +} + +inline void session::erase(const shared_bytes& key) { + auto key_slice = rocksdb::Slice{ key.data(), key.size() }; + auto status = m_db->Delete(m_write_options, column_family_(), key_slice); +} + +inline void session::clear() {} + +template +const std::pair>, std::unordered_set> +session::read_(const Iterable& keys) { + auto not_found = std::unordered_set{}; + auto key_slices = std::vector{}; + + for (const auto& key : keys) { + key_slices.emplace_back(key.data(), key.size()); + not_found.emplace(key); + } + + auto values = std::vector{}; + values.reserve(key_slices.size()); + auto status = m_db->MultiGet(m_read_options, { key_slices.size(), column_family_() }, key_slices, &values); + + auto kvs = std::vector>{}; + kvs.reserve(key_slices.size()); + + for (size_t i = 0; i < values.size(); ++i) { + if (status[i].code() != rocksdb::Status::Code::kOk) { + continue; + } + + auto key = shared_bytes(key_slices[i].data(), key_slices[i].size()); + not_found.erase(key); + kvs.emplace_back(key, shared_bytes(values[i].data(), values[i].size())); + } + + return { std::move(kvs), std::move(not_found) }; +} + +template +const std::pair>, std::unordered_set> +session::read(const Iterable& keys) { + return read_(keys); +} + +template +void session::write(const Iterable& key_values) { + auto batch = rocksdb::WriteBatch{ 1024 * 1024 }; + + for (const auto& kv : key_values) { + batch.Put(column_family_(), { kv.first.data(), kv.first.size() }, { kv.second.data(), kv.second.size() }); + } + + auto status = m_db->Write(m_write_options, &batch); +} + +template +void session::erase(const Iterable& keys) { + for (const auto& key : keys) { + auto key_slice = rocksdb::Slice{ key.data(), key.size() }; + auto status = m_db->Delete(m_write_options, column_family_(), key_slice); + } +} + +template +void session::write_to(Other_data_store& ds, const Iterable& keys) { + auto [found, not_found] = read_(keys); + ds.write(found); +} + +template +void session::read_from(Other_data_store& ds, const Iterable& keys) { + auto [found, not_found] = ds.read(keys); + write(found); +} + +template +using rocks_iterator_alias = typename session::template rocks_iterator; + +template +typename session::iterator session::make_iterator_(const Predicate& setup) const { + rocksdb::Iterator* rit = nullptr; + int64_t index = -1; + if (!m_free_list.empty()) { + index = m_free_list.back(); + m_free_list.pop_back(); + rit = m_iterators[index].get(); + rit->Refresh(); + } else { + rit = m_db->NewIterator(m_iterator_read_options, column_family_()); + } + setup(*rit); + + return { *const_cast*>(this), *rit, index }; +} + +inline typename session::iterator session::find(const shared_bytes& key) { + auto predicate = [&](auto& it) { + auto key_slice = rocksdb::Slice{ key.data(), key.size() }; + it.Seek(key_slice); + if (it.Valid() && it.key().compare(key_slice) != 0) { + // Get an invalid iterator + it.Refresh(); + } + }; + return make_iterator_(predicate); +} + +inline typename session::iterator session::begin() { + return make_iterator_([](auto& it) { it.SeekToFirst(); }); +} + +inline typename session::iterator session::end() { + return make_iterator_([](auto& it) {}); +} + +inline typename session::iterator session::lower_bound(const shared_bytes& key) { + return make_iterator_([&](auto& it) { it.Seek(rocksdb::Slice{ key.data(), key.size() }); }); +} + +inline void session::flush() { + rocksdb::FlushOptions op; + op.allow_write_stall = true; + op.wait = true; + m_db->Flush(op); +} + +inline void session::destroy(const std::string& db_name) { + rocksdb::Options options; + rocksdb::DestroyDB(db_name, options); +} + +inline rocksdb::WriteOptions& session::write_options() { return m_write_options; } + +inline const rocksdb::WriteOptions& session::write_options() const { return m_write_options; } + +inline rocksdb::ReadOptions& session::read_options() { return m_read_options; } + +inline const rocksdb::ReadOptions& session::read_options() const { return m_read_options; } + +inline std::shared_ptr& session::column_family() { return m_column_family; } + +inline std::shared_ptr session::column_family() const { + return m_column_family; +} + +inline rocksdb::ColumnFamilyHandle* session::column_family_() const { + if (m_column_family) { + return m_column_family.get(); + } + + if (m_db) { + return m_db->DefaultColumnFamily(); + } + + return nullptr; +} + +template +session::rocks_iterator::rocks_iterator(const rocks_iterator& it) + : rocks_iterator{ it.m_iterator->Valid() ? it.m_session->find(it.key()) : it.m_session->end() } {} + +template +session::rocks_iterator::rocks_iterator(rocks_iterator&& it) + : m_session{ std::exchange(it.m_session, nullptr) }, m_iterator{ std::exchange(it.m_iterator, nullptr) }, m_index{ + std::exchange(it.m_index, -1) + } {} + +template +session::rocks_iterator::rocks_iterator(session& session, rocksdb::Iterator& rit, + int64_t index) + : m_session{ &session }, m_iterator{ &rit }, m_index{ index } {} + +template +rocks_iterator_alias& +session::rocks_iterator::operator=(const rocks_iterator& it) { + if (this == &it) { + return *this; + } + + reset(); + + m_session = it.m_session; + if (m_session) { + return *this; + } + + auto new_iterator = rocks_iterator{}; + if (it.m_iterator->Valid()) { + new_iterator = m_session->find(it.key()); + } else { + new_iterator = std::end(*m_session); + } + m_iterator = std::exchange(new_iterator.m_iterator, nullptr); + m_index = std::exchange(new_iterator.m_index, -1); + + return *this; +} + +template +rocks_iterator_alias& +session::rocks_iterator::operator=(rocks_iterator&& it) { + if (this == &it) { + return *this; + } + + reset(); + + m_session = std::exchange(it.m_session, nullptr); + m_iterator = std::exchange(it.m_iterator, nullptr); + m_index = std::exchange(it.m_index, -1); + + return *this; +} + +template +session::rocks_iterator::~rocks_iterator() { + reset(); +} + +template +void session::rocks_iterator::reset() { + if (m_index > -1) { + m_session->m_free_list.push_back(m_index); + } else if (m_iterator) { + delete m_iterator; + } + m_iterator = nullptr; + m_index = -1; + m_session = nullptr; +} + +template +rocks_iterator_alias& session::rocks_iterator::operator++() { + if (!m_iterator->Valid()) { + m_iterator->SeekToFirst(); + } else { + m_iterator->Next(); + } + return *this; +} + +template +rocks_iterator_alias& session::rocks_iterator::operator--() { + if (!m_iterator->Valid()) { + // This means we are at the end iterator and we are iterating backwards. + m_iterator->SeekToLast(); + } else { + m_iterator->Prev(); + } + return *this; +} + +template +shared_bytes session::rocks_iterator::key() const { + if (!m_iterator->Valid()) { + return shared_bytes{}; + } + + auto key_slice = m_iterator->key(); + return shared_bytes(key_slice.data(), key_slice.size()); +} + +template +bool session::rocks_iterator::deleted() const { + return false; +} + +template +typename rocks_iterator_alias::value_type +session::rocks_iterator::operator*() const { + if (!m_iterator->Valid()) { + return std::pair{ shared_bytes{}, std::optional{} }; + } + + auto key_slice = m_iterator->key(); + auto value = m_iterator->value(); + return std::pair{ shared_bytes(key_slice.data(), key_slice.size()), + std::optional{ shared_bytes(value.data(), value.size()) } }; +} + +template +typename rocks_iterator_alias::value_type +session::rocks_iterator::operator->() const { + if (!m_iterator->Valid()) { + return std::pair{ shared_bytes{}, std::optional{} }; + } + + auto key_slice = m_iterator->key(); + auto value = m_iterator->value(); + return std::pair{ shared_bytes(key_slice.data(), key_slice.size()), + std::optional{ shared_bytes(value.data(), value.size()) } }; +} + +template +bool session::rocks_iterator::operator==(const rocks_iterator& other) const { + if (!m_iterator->Valid()) { + return !other.m_iterator->Valid(); + } + + if (!other.m_iterator->Valid()) { + return !m_iterator->Valid(); + } + + return m_iterator->key().compare(other.m_iterator->key()) == 0; +} + +template +bool session::rocks_iterator::operator!=(const rocks_iterator& other) const { + return !(*this == other); +} + +} // namespace eosio::session diff --git a/libraries/chain_kv/include/b1/session/session.hpp b/libraries/chain_kv/include/b1/session/session.hpp new file mode 100644 index 00000000000..87886756e59 --- /dev/null +++ b/libraries/chain_kv/include/b1/session/session.hpp @@ -0,0 +1,1031 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace eosio::session { + +template +struct overloaded : Ts... { + using Ts::operator()...; +}; +template +overloaded(Ts...) -> overloaded; + +/// \brief Defines a session for reading/write data to a cache and persistent data store. +/// \tparam Parent The parent type of this session +/// \remarks Specializations of this type can be created to create new parent types that +/// modify a different data store. For an example refer to the rocks_session type in this folder. +template +class session { + public: + struct value_state { + /// Indicates if the next key, in lexicographical order, is within the cache. + bool next_in_cache{ false }; + /// Indicates if the previous key, in lexicographical order, is within the cache. + bool previous_in_cache{ false }; + /// Indicates if this key/value pair has been deleted. + bool deleted{ false }; + /// Indictes if this key's value has been updated. + bool updated{ false }; + /// The current version of the key's value. + uint64_t version{ 0 }; + /// The key's value + shared_bytes value; + }; + + using type = session; + using parent_type = Parent; + using cache_type = std::map; + using parent_variant_type = std::variant; + + friend Parent; + + /// \brief Defines a key lexicographically ordered, cyclical iterator for traversing a session (which includes + /// parents). + template + class session_iterator { + public: + using difference_type = typename Iterator_traits::difference_type; + using value_type = typename Iterator_traits::value_type; + using pointer = typename Iterator_traits::pointer; + using reference = typename Iterator_traits::reference; + using iterator_category = typename Iterator_traits::iterator_category; + friend session; + + public: + session_iterator(session* active_session, typename Iterator_traits::cache_iterator it, uint64_t version); + session_iterator() = default; + session_iterator(const session_iterator& it) = default; + session_iterator(session_iterator&&) = default; + + session_iterator& operator=(const session_iterator& it) = default; + session_iterator& operator=(session_iterator&&) = default; + + session_iterator& operator++(); + session_iterator& operator--(); + value_type operator*() const; + value_type operator->() const; + bool operator==(const session_iterator& other) const; + bool operator!=(const session_iterator& other) const; + + /// \brief Indicates if the key this iterator refers to has been deleted in the cache or database. + /// \return True if the key has been deleted, false otherwise. + /// \remarks Keys can be deleted while there are iterator instances refering to them and the iterator + /// effectively becomes invalidated. + bool deleted() const; + + /// \brief Returns the current key that this iterator is positioned on. + /// \remarks Prefer calling this over the dereference operator if all you want is the key value. + /// This method does not read any values from the session heirachary (which would incur unncessary overhead if you + /// don't care about the value.) + const shared_bytes& key() const; + + protected: + /// \brief Moves this iterator to another value. + /// \tparam Test_predicate A functor with the following signature + /// bool(typename Iterator_traits::cache_iterator&) which tests if the key being moved to exists in + /// the cache. This method should return true if the desired key is in the cache, false otherwise. + /// \tparam Move_predicate A functor with the following signature + /// void(typename Iterator_traits::cache_iterator&) which moves the iterator to the desired key in + /// the cache. + /// \tparam Cache_update A functor with the followig signature + /// bool(typename Iterator_traits::cache_iterator&) which performs any needed updates to the cache to + /// prepare to move to the desired key. This method should return true if the cache was updated, false otherwise. + /// \param test The Test_predicate functor instance. + /// \param move The Move_predicate functor instance. + /// \param update_cache The Cache_update functor instance. + template + void move_(const Test_predicate& test, const Move_predicate& move, Cache_update& update_cache); + + /// \brief Moves the iterator to the next key in lexicographical order. + void move_next_(); + + /// \brief Moves the iterator to the previous key in lexicographical order. + void move_previous_(); + + private: + /// The version of the key's value that this iterator is concerned with. + /// \remarks If the version is different than the iterator's version, then this indicates the key was deleted and + /// this iterator is effectively invalidated. + uint64_t m_iterator_version{ 0 }; + typename Iterator_traits::cache_iterator m_active_iterator; + session* m_active_session{ nullptr }; + }; + + struct iterator_traits { + using difference_type = std::ptrdiff_t; + using value_type = std::pair>; + using pointer = value_type*; + using reference = value_type&; + using iterator_category = std::bidirectional_iterator_tag; + using cache_iterator = typename cache_type::iterator; + }; + using iterator = session_iterator; + + public: + session() = default; + + /// \brief Constructs a child session of the templated parent type. + session(Parent& parent); + + /// \brief Constructs a child session from another instance of the same session type. + explicit session(session& parent, std::nullptr_t); + session(const session&) = delete; + session(session&& other); + ~session(); + + session& operator=(const session&) = delete; + + session& operator=(session&& other); + + parent_variant_type parent() const; + + /// \brief Returns the set of keys that have been updated in this session. + std::unordered_set updated_keys() const; + + /// \brief Returns the set of keys that have been deleted in this session. + std::unordered_set deleted_keys() const; + + /// \brief Attaches a new parent to the session. + void attach(Parent& parent); + + /// \brief Attaches a new parent to the session. + void attach(session& parent); + + /// \brief Detaches from the parent. + /// \remarks This is very similar to an undo. + void detach(); + + /// \brief Discards the changes in this session and detachs the session from its parent. + void undo(); + /// \brief Commits the changes in this session into its parent. + void commit(); + + std::optional read(const shared_bytes& key); + void write(const shared_bytes& key, const shared_bytes& value); + bool contains(const shared_bytes& key); + void erase(const shared_bytes& key); + void clear(); + + /// \brief Reads a batch of keys from this session. + /// \param keys A type that supports iteration and returns in its iterator a shared_bytes type representing the key. + template + const std::pair>, std::unordered_set> + read(const Iterable& keys); + + /// \brief Writes a batch of key/value pairs into this session. + /// \param key_values A type that supports iteration and returns in its iterator a pair containing shared_bytes + /// instances that represents a key and a value. + template + void write(const Iterable& key_values); + + /// \brief Erases a batch of keys from this session. + /// \param keys A type that supports iteration and returns in its iterator a shared_bytes type representing the key. + template + void erase(const Iterable& keys); + + /// \brief Writes a batch of key/values from this session into another container. + /// \param ds The container to write into. The type must implement a batch write method like the one defined in this + /// type. \param keys A type that supports iteration and returns in its iterator a shared_bytes type representing the + /// key. + template + void write_to(Other_data_store& ds, const Iterable& keys); + + /// \brief Reads a batch of keys from another container into this session. + /// \param ds The container to read from. This type must implement a batch read method like the one defined in this + /// type. \param keys A type that supports iteration and returns in its iterator a shared_bytes type representing the + /// key. + template + void read_from(Other_data_store& ds, const Iterable& keys); + + /// \brief Returns an iterator to the key, or the end iterator if the key is not in the cache or session heirarchy. + /// \param key The key to search for. + /// \return An iterator to the key if found, the end iterator otherwise. + iterator find(const shared_bytes& key); + iterator begin(); + iterator end(); + + /// \brief Returns an iterator to the first key that is not less than, in lexicographical order, the given key. + /// \param key The key to search on. + /// \return An iterator to the first key that is not less than the given key, or the end iterator if there is no key + /// that matches that criteria. + iterator lower_bound(const shared_bytes& key); + + private: + /// \brief Sets the lower/upper bounds of the session's cache based on the parent's cache lower/upper bound + /// \remarks This is only invoked when constructing a session with a parent. This method prepares the iterator cache + /// by priming it with a lower/upper bound for iteration. These bounds can be changed, though, within this session + /// through adding and deleting keys through this sesion. + void prime_cache_(); + + /// \brief Searches the session heirarchy for the key that is next, in lexicographical order, from the given key. + /// \param it An iterator pointing to the current key. + /// \param pit An iterator pointing to the lower_bound of the key in the parent session. + /// \param pend An end iterator of the parent session. + /// \remarks This method is only called by update_iterator_cache_. + template + void next_key_(It& it, Parent_it& pit, Parent_it& pend); + + /// \brief Searches the session heirarchy for the key that is previous, in lexicographical order, from the given key. + /// \param it An iterator pointing to the current key. + /// \param pit An iterator pointing to the lower_bound of the key in the parent session. + /// \param pend An end iterator of the parent session. + /// \remarks This method is only called by update_iterator_cache_. + template + void previous_key_(It& it, Parent_it& pit, Parent_it& pbegin, Parent_it& pend); + + /// \brief Updates the cache with the previous and next keys, in lexicographical order, of the given key. + /// \param key The key to search on. + /// \remarks This method is invoked when reading, writing, erasing on this session and when iterating over the + /// session. + typename cache_type::iterator update_iterator_cache_(const shared_bytes& key); + + /// \brief Increments the given cache iterator until an iterator is found that isn't pointing to a deleted key. + /// \param it The cache iterator to increment. + /// \param end An end iterator of the cache. + template + It& first_not_deleted_in_iterator_cache_(It& it, const It& end, bool& previous_in_cache) const; + + template + It& first_not_deleted_in_iterator_cache_(It& it, const It& end) const; + + private: + parent_variant_type m_parent{ static_cast(nullptr) }; + cache_type m_cache; +}; + +template +typename session::parent_variant_type session::parent() const { + return m_parent; +} + +template +void session::prime_cache_() { + // Get the bounds of the parent cache and use those to + // seed the cache of this session. + auto update = [&](const auto& key, const auto& value) { + auto it = m_cache.emplace(key, value_state{}); + if (it.second) { + it.first->second.value = value; + } + }; + + std::visit( + [&](auto* p) { + auto begin = std::begin(*p); + auto end = std::end(*p); + if (begin == end) { + return; + } + update(begin.key(), (*begin).second.value()); + --end; + update(end.key(), (*end).second.value()); + }, + m_parent); +} + +template +void session::clear() { + m_cache.clear(); +} + +template +session::session(Parent& parent) : m_parent{ &parent } { + attach(parent); +} + +template +session::session(session& parent, std::nullptr_t) : m_parent{ &parent } { + attach(parent); +} + +template +session::session(session&& other) : m_parent{ std::move(other.m_parent) }, m_cache{ std::move(other.m_cache) } { + session* null_parent = nullptr; + other.m_parent = null_parent; +} + +template +session& session::operator=(session&& other) { + if (this == &other) { + return *this; + } + + m_parent = std::move(other.m_parent); + m_cache = std::move(other.m_cache); + + session* null_parent = nullptr; + other.m_parent = null_parent; + + return *this; +} + +template +session::~session() { + commit(); + undo(); +} + +template +void session::undo() { + detach(); + clear(); +} + +template +template +void session::previous_key_(It& it, Parent_it& pit, Parent_it& pbegin, Parent_it& pend) { + if (it->first) { + if (pit != pbegin) { + --pit; + if (pit != pend) { + auto cit = m_cache.emplace(pit.key(), value_state{}); + if (cit.second) { + cit.first->second.value = *(*pit).second; + } + } + ++pit; + } + + auto previous_it = it; + if (previous_it != std::begin(m_cache)) { + --previous_it; + previous_it->second.next_in_cache = true; + it->second.previous_in_cache = true; + } + } +} + +template +template +void session::next_key_(It& it, Parent_it& pit, Parent_it& pend) { + if (it->first) { + bool decrement = false; + if (pit.key() == it->first) { + ++pit; + decrement = true; + } + if (pit != pend) { + auto cit = m_cache.emplace(pit.key(), value_state{}); + if (cit.second) { + cit.first->second.value = *(*pit).second; + } + } + if (decrement) { + --pit; + } + + auto next_it = it; + ++next_it; + if (next_it != std::end(m_cache)) { + next_it->second.previous_in_cache = true; + it->second.next_in_cache = true; + } + } +} + +template +typename session::cache_type::iterator session::update_iterator_cache_(const shared_bytes& key) { + auto result = m_cache.emplace(key, value_state{}); + auto& it = result.first; + + if (result.second) { + // The two keys that this new key is being inserted inbetween might already be in a global lexicographical order. + // If so, that means we already know the global order of this new key. + if (it != std::begin(m_cache)) { + auto previous = it; + --previous; + if (previous->second.next_in_cache) { + it->second.previous_in_cache = true; + it->second.next_in_cache = true; + return it; + } + } else { + auto end = std::end(m_cache); + if (it != end) { + auto next = it; + ++next; + if (next != end && next->second.previous_in_cache) { + it->second.next_in_cache = true; + it->second.previous_in_cache = true; + return it; + } + } + } + + // ...otherwise we have to search through the session heirarchy to find the previous and next keys + // of the given key. + if (!it->second.next_in_cache || !it->second.previous_in_cache) { + std::visit( + [&](auto* p) { + auto pit = p->lower_bound(key); + auto end = std::end(*p); + if (!it->second.next_in_cache) { + next_key_(it, pit, end); + } + if (!it->second.previous_in_cache) { + auto begin = std::begin(*p); + previous_key_(it, pit, begin, end); + } + }, + m_parent); + } + } + + return it; +} + +template +std::unordered_set session::updated_keys() const { + auto results = std::unordered_set{}; + for (const auto& it : m_cache) { + if (it.second.updated) { + results.emplace(it.first); + } + } + return results; +} + +template +std::unordered_set session::deleted_keys() const { + auto results = std::unordered_set{}; + for (const auto& it : m_cache) { + if (it.second.deleted) { + results.emplace(it.first); + } + } + return results; +} + +template +void session::attach(Parent& parent) { + m_parent = &parent; + prime_cache_(); +} + +template +void session::attach(session& parent) { + m_parent = &parent; + prime_cache_(); +} + +template +void session::detach() { + session* null_parent = nullptr; + m_parent = null_parent; +} + +template +void session::commit() { + if (m_cache.empty()) { + // Nothing to commit. + return; + } + + auto write_through = [&](auto& ds) { + auto deletes = std::unordered_set{}; + auto updates = std::unordered_map{}; + + for (const auto& p : m_cache) { + if (p.second.deleted) { + deletes.emplace(p.first); + } else if (p.second.updated) { + updates.emplace(p.first, p.second.value); + } + } + + if (deletes.size() > 0) { + ds.erase(deletes); + } + + if (updates.size() > 0) { + ds.write(updates); + } + + clear(); + }; + + std::visit( + [&](auto* p) { + if (!p) { + return; + } + write_through(*p); + }, + m_parent); +} + +template +std::optional session::read(const shared_bytes& key) { + // Find the key within the session. + // Check this level first and then traverse up to the parent to see if this key/value + // has been read and/or update. + auto it = m_cache.find(key); + if (it != std::end(m_cache) && it->second.deleted) { + // key has been deleted at this level. + return {}; + } + + if (it != std::end(m_cache) && it->second.value) { + return it->second.value; + } + + auto value = std::optional{}; + std::visit( + [&](auto* p) { + if (p) { + value = p->read(key); + } + }, + m_parent); + + if (value) { + // Update the "iterator cache". + auto it = update_iterator_cache_(key); + it->second.value = *value; + } + + return value; +} + +template +void session::write(const shared_bytes& key, const shared_bytes& value) { + auto it = update_iterator_cache_(key); + it->second.value = value; + it->second.deleted = false; + it->second.updated = true; +} + +template +bool session::contains(const shared_bytes& key) { + // Traverse the heirarchy to see if this session (and its parent session) + // has already read the key into memory. + + auto it = m_cache.find(key); + if (it != std::end(m_cache) && it->second.deleted) { + // deleted. + return false; + } + + if (it != std::end(m_cache) && it->second.value) { + return true; + } + + return std::visit( + [&](auto* p) { + auto value = p->read(key); + if (value) { + auto it = update_iterator_cache_(key); + it->second.value = std::move(*value); + return true; + } + return false; + }, + m_parent); +} + +template +void session::erase(const shared_bytes& key) { + auto it = update_iterator_cache_(key); + it->second.deleted = true; + it->second.updated = false; + ++it->second.version; +} + +template +template +const std::pair>, std::unordered_set> +session::read(const Iterable& keys) { + auto not_found = std::unordered_set{}; + auto kvs = std::vector>{}; + + for (const auto& key : keys) { + auto value = read(key); + if (value != shared_bytes{}) { + kvs.emplace_back(key, value); + } else { + not_found.emplace(key); + } + } + + return { std::move(kvs), std::move(not_found) }; +} + +template +template +void session::write(const Iterable& key_values) { + // Currently the batch write will just iteratively call the non batch write + for (const auto& kv : key_values) { write(kv.first, kv.second); } +} + +template +template +void session::erase(const Iterable& keys) { + // Currently the batch erase will just iteratively call the non batch erase + for (const auto& key : keys) { erase(key); } +} + +template +template +void session::write_to(Other_data_store& ds, const Iterable& keys) { + auto results = std::vector>{}; + for (const auto& key : keys) { + auto value = read(key); + if (value != shared_bytes{}) { + results.emplace_back(shared_bytes(key.data(), key.size()), shared_bytes(value.data(), value.size())); + } + } + ds.write(results); +} + +template +template +void session::read_from(Other_data_store& ds, const Iterable& keys) { + ds.write_to(*this, keys); +} + +template +template +It& session::first_not_deleted_in_iterator_cache_(It& it, const It& end, bool& previous_in_cache) const { + auto previous_known = true; + auto update_previous_flag = [&](auto& it) { + if (previous_known) { + previous_known = it->second.previous_in_cache; + } + }; + + while (it != end && it->second.deleted) { + update_previous_flag(it); + ++it; + } + update_previous_flag(it); + previous_in_cache = previous_known; + return it; +} + +template +template +It& session::first_not_deleted_in_iterator_cache_(It& it, const It& end) const { + while (it != end) { + auto find_it = m_cache.find(it.key()); + if (find_it == std::end(m_cache) || !find_it->second.deleted) { + return it; + } + ++it; + } + + return it; +} + +template +typename session::iterator session::find(const shared_bytes& key) { + auto version = uint64_t{ 0 }; + auto end = std::end(m_cache); + auto it = m_cache.find(key); + if (it == end) { + // We didn't find the key in our cache, ask the parent if he has one. + std::visit( + [&](auto* p) { + auto pit = p->find(key); + if (pit != std::end(*p)) { + auto result = m_cache.emplace(key, value_state{}); + it = result.first; + if (result.second) { + it->second.value = *(*pit).second; + } + } + }, + m_parent); + } + + if (it != end) { + version = it->second.version; + if (it->second.deleted) { + it = std::move(end); + } + } + return { const_cast(this), std::move(it), version }; +} + +template +typename session::iterator session::begin() { + auto end = std::end(m_cache); + auto begin = std::begin(m_cache); + auto it = begin; + auto version = uint64_t{ 0 }; + + auto previous_in_cache = true; + first_not_deleted_in_iterator_cache_(it, end, previous_in_cache); + + if (it == end || (it != begin && !previous_in_cache)) { + // We have a begin iterator in this session, but we don't have enough + // information to determine if that iterator is globally the begin iterator. + // We need to ask the parent for its begin iterator and compare the two + // to see which comes first lexicographically. + auto pending_key = shared_bytes{}; + if (it != end) { + pending_key = (*it).first; + } + std::visit( + [&](auto* p) { + auto pit = std::begin(*p); + auto pend = std::end(*p); + first_not_deleted_in_iterator_cache_(pit, pend); + if (pit != pend && pit.key() < pending_key) { + auto result = m_cache.emplace(pit.key(), value_state{}); + it = result.first; + if (result.second) { + it->second.value = *(*pit).second; + } + } + }, + m_parent); + } + + if (it != end) { + version = it->second.version; + } + return { const_cast(this), std::move(it), version }; +} + +template +typename session::iterator session::end() { + return { const_cast(this), std::end(m_cache), 0 }; +} + +template +typename session::iterator session::lower_bound(const shared_bytes& key) { + auto version = uint64_t{ 0 }; + auto end = std::end(m_cache); + auto it = m_cache.lower_bound(key); + + auto previous_in_cache = true; + first_not_deleted_in_iterator_cache_(it, end, previous_in_cache); + + if (it == end || ((*it).first != key && !previous_in_cache)) { + // So either: + // 1. We didn't find a key in the iterator cache (pending_key is invalid). + // 2. The pending_key is not exactly the key and the found key in the iterator cache currently doesn't know what + // its previous key is. + auto pending_key = shared_bytes{}; + if (it != end) { + pending_key = (*it).first; + } + std::visit( + [&](auto* p) { + auto pit = p->lower_bound(key); + auto pend = std::end(*p); + first_not_deleted_in_iterator_cache_(pit, pend); + if (pit != pend) { + if (!pending_key || pit.key() < pending_key) { + auto result = m_cache.emplace(pit.key(), value_state{}); + it = result.first; + if (result.second) { + it->second.value = *(*pit).second; + } + } + } + }, + m_parent); + } + if (it != end) { + version = it->second.version; + if (it->second.deleted) { + it = std::move(end); + } + } + return { const_cast(this), std::move(it), version }; +} + +template +template +session::session_iterator::session_iterator(session* active_session, + typename Iterator_traits::cache_iterator it, + uint64_t version) + : m_iterator_version{ version }, m_active_iterator{ std::move(it) }, m_active_session{ active_session } {} + +template +template +template +void session::session_iterator::move_(const Test_predicate& test, const Move_predicate& move, + Cache_update& update_cache) { + do { + if (m_active_iterator != std::end(m_active_session->m_cache) && !test(m_active_iterator)) { + // Force an update to see if we pull in a next or previous key from the current key. + if (!update_cache(m_active_iterator)) { + // The test still fails. We are at the end. + m_active_iterator = std::end(m_active_session->m_cache); + break; + } + } + // Move to the next iterator in the cache. + move(m_active_iterator); + if (m_active_iterator == std::end(m_active_session->m_cache) || !m_active_iterator->second.deleted) { + // We either found a key that hasn't been deleted or we hit the end of the iterator cache. + break; + } + } while (true); +} + +template +template +void session::session_iterator::move_next_() { + auto move = [](auto& it) { ++it; }; + auto test = [](auto& it) { return it->second.next_in_cache; }; + auto update_cache = [&](auto& it) mutable { + auto value = std::optional{}; + + auto pending_key = eosio::session::shared_bytes{}; + auto end = std::end(m_active_session->m_cache); + if (it != end) { + ++it; + if (it != end) { + pending_key = it->first; + } + --it; + } + + auto key = std::visit( + [&](auto* p) { + auto pit = p->lower_bound(it->first); + if (pit != std::end(*p) && pit.key() == it->first) { + ++pit; + } + value = (*pit).second; + return pit.key(); + }, + m_active_session->m_parent); + + // We have two candidates for the next key. + // 1. The next key in order in this sessions cache. + // 2. The next key in lexicographical order retrieved from the sessions parent. + // Choose which one it is. + if (!key || (pending_key && pending_key < key)) { + key = pending_key; + } + + if (key) { + auto nit = m_active_session->m_cache.emplace(key, value_state{}); + nit.first->second.previous_in_cache = true; + it->second.next_in_cache = true; + if (nit.second) { + nit.first->second.value = std::move(*value); + } + return true; + } + return false; + }; + + if (m_active_iterator == std::end(m_active_session->m_cache)) { + m_active_iterator = std::begin(m_active_session->m_cache); + } else { + move_(test, move, update_cache); + } +} + +template +template +void session::session_iterator::move_previous_() { + auto move = [](auto& it) { --it; }; + auto test = [&](auto& it) { + if (it != std::end(m_active_session->m_cache)) { + return it->second.previous_in_cache; + } + return true; + }; + auto update_cache = [&](auto& it) mutable { + auto value = std::optional{}; + + auto pending_key = eosio::session::shared_bytes{}; + if (it != std::begin(m_active_session->m_cache)) { + --it; + pending_key = it->first; + ++it; + } + + auto key = std::visit( + [&](auto* p) { + auto pit = p->lower_bound(it->first); + if (pit != std::begin(*p)) { + --pit; + } else { + return eosio::session::shared_bytes{}; + } + + value = (*pit).second; + return pit.key(); + }, + m_active_session->m_parent); + + // We have two candidates to consider. + // 1. The key returned by decrementing the iterator on this session's cache. + // 2. The key returned by calling lower_bound on the parent of this session. + // We want the larger of the two. + if (!key || (pending_key && pending_key > key)) { + key = pending_key; + } + + if (key) { + auto nit = m_active_session->m_cache.emplace(key, value_state{}); + nit.first->second.next_in_cache = true; + it->second.previous_in_cache = true; + if (nit.second) { + nit.first->second.value = std::move(*value); + } + return true; + } + return false; + }; + + if (m_active_iterator == std::begin(m_active_session->m_cache)) { + m_active_iterator = std::end(m_active_session->m_cache); + } else { + move_(test, move, update_cache); + } +} + +template +template +typename session::template session_iterator& +session::session_iterator::operator++() { + move_next_(); + return *this; +} + +template +template +typename session::template session_iterator& +session::session_iterator::operator--() { + move_previous_(); + return *this; +} + +template +template +bool session::session_iterator::deleted() const { + if (m_active_iterator == std::end(m_active_session->m_cache)) { + return false; + } + + return m_active_iterator->second.deleted || m_iterator_version != m_active_iterator->second.version; +} + +template +template +const shared_bytes& session::session_iterator::key() const { + if (m_active_iterator == std::end(m_active_session->m_cache)) { + static auto empty = shared_bytes{}; + return empty; + } + return m_active_iterator->first; +} + +template +template +typename session::template session_iterator::value_type +session::session_iterator::operator*() const { + if (m_active_iterator == std::end(m_active_session->m_cache)) { + return std::pair{ shared_bytes{}, std::optional{} }; + } + return std::pair{ m_active_iterator->first, m_active_iterator->second.value }; +} + +template +template +typename session::template session_iterator::value_type +session::session_iterator::operator->() const { + if (m_active_iterator == std::end(m_active_session->m_cache)) { + return std::pair{ shared_bytes{}, std::optional{} }; + } + return std::pair{ m_active_iterator->first, m_active_iterator->second.value }; +} + +template +template +bool session::session_iterator::operator==(const session_iterator& other) const { + auto end = std::end(m_active_session->m_cache); + if (m_active_iterator == end && m_active_iterator == other.m_active_iterator) { + return true; + } + if (other.m_active_iterator == end) { + return false; + } + return this->m_active_iterator == other.m_active_iterator; +} + +template +template +bool session::session_iterator::operator!=(const session_iterator& other) const { + return !(*this == other); +} + +} // namespace eosio::session diff --git a/libraries/chain_kv/include/b1/session/session_variant.hpp b/libraries/chain_kv/include/b1/session/session_variant.hpp new file mode 100644 index 00000000000..85583c72d67 --- /dev/null +++ b/libraries/chain_kv/include/b1/session/session_variant.hpp @@ -0,0 +1,292 @@ +#pragma once + +#include + +namespace eosio::session { + +template +class session_variant { + public: + template + class session_variant_iterator { + public: + using difference_type = typename Iterator_traits::difference_type; + using value_type = typename Iterator_traits::value_type; + using pointer = typename Iterator_traits::pointer; + using reference = typename Iterator_traits::reference; + using iterator_category = typename Iterator_traits::iterator_category; + friend session_variant; + + public: + template + session_variant_iterator(U iterator); + session_variant_iterator() = default; + session_variant_iterator(const session_variant_iterator& it) = default; + session_variant_iterator(session_variant_iterator&&) = default; + + session_variant_iterator& operator=(const session_variant_iterator& it) = default; + session_variant_iterator& operator=(session_variant_iterator&&) = default; + + session_variant_iterator& operator++(); + session_variant_iterator& operator--(); + value_type operator*() const; + bool operator==(const session_variant_iterator& other) const; + bool operator!=(const session_variant_iterator& other) const; + + bool deleted() const; + const shared_bytes key() const; + + private: + std::variant m_holder; + }; + + struct iterator_traits { + using difference_type = std::ptrdiff_t; + using value_type = std::pair>; + using pointer = value_type*; + using reference = value_type&; + using iterator_category = std::bidirectional_iterator_tag; + }; + using iterator = session_variant_iterator; + + public: + session_variant() = default; + template + session_variant(U& session, std::nullptr_t); + session_variant(const session_variant&) = default; + session_variant(session_variant&& other) = default; + ~session_variant() = default; + + session_variant& operator=(const session_variant&) = default; + + session_variant& operator=(session_variant&& other) = default; + + /// \brief Returns the set of keys that have been updated in this session. + std::unordered_set updated_keys() const; + + /// \brief Returns the set of keys that have been deleted in this session. + std::unordered_set deleted_keys() const; + + /// \brief Discards the changes in this session and detachs the session from its parent. + void undo(); + /// \brief Commits the changes in this session into its parent. + void commit(); + + std::optional read(const shared_bytes& key); + void write(const shared_bytes& key, const shared_bytes& value); + bool contains(const shared_bytes& key); + void erase(const shared_bytes& key); + void clear(); + + /// \brief Reads a batch of keys from this session. + /// \param keys A type that supports iteration and returns in its iterator a shared_bytes type representing the key. + template + const std::pair>, std::unordered_set> + read(const Iterable& keys); + + /// \brief Writes a batch of key/value pairs into this session. + /// \param key_values A type that supports iteration and returns in its iterator a pair containing shared_bytes + /// instances that represents a key and a value. + template + void write(const Iterable& key_values); + + /// \brief Erases a batch of keys from this session. + /// \param keys A type that supports iteration and returns in its iterator a shared_bytes type representing the key. + template + void erase(const Iterable& keys); + + /// \brief Writes a batch of key/values from this session into another container. + /// \param ds The container to write into. The type must implement a batch write method like the one defined in this + /// type. \param keys A type that supports iteration and returns in its iterator a shared_bytes type representing the + /// key. + template + void write_to(Other_data_store& ds, const Iterable& keys); + + /// \brief Reads a batch of keys from another container into this session. + /// \param ds The container to read from. This type must implement a batch read method like the one defined in this + /// type. \param keys A type that supports iteration and returns in its iterator a shared_bytes type representing the + /// key. + template + void read_from(Other_data_store& ds, const Iterable& keys); + + /// \brief Returns an iterator to the key, or the end iterator if the key is not in the cache or session heirarchy. + /// \param key The key to search for. + /// \return An iterator to the key if found, the end iterator otherwise. + iterator find(const shared_bytes& key); + iterator begin(); + iterator end(); + + /// \brief Returns an iterator to the first key that is not less than, in lexicographical order, the given key. + /// \param key The key to search on. + /// \return An iterator to the first key that is not less than the given key, or the end iterator if there is no key + /// that matches that criteria. + iterator lower_bound(const shared_bytes& key); + + std::variant holder(); + + private: + std::variant m_holder; +}; + +template +template +session_variant::session_variant(U& session, std::nullptr_t) : m_holder{ &session } {} + +template +std::unordered_set session_variant::updated_keys() const { + return std::visit([&](auto* session) { return session->updated_keys(); }, m_holder); +} + +template +std::unordered_set session_variant::deleted_keys() const { + return std::visit([&](auto* session) { return session->deleted_keys(); }, m_holder); +} + +template +std::variant session_variant::holder() { + return m_holder; +} + +template +void session_variant::undo() { + std::visit([&](auto* session) { return session->undo(); }, m_holder); +} + +template +void session_variant::commit() { + std::visit([&](auto* session) { return session->commit(); }, m_holder); +} + +template +std::optional session_variant::read(const shared_bytes& key) { + return std::visit([&](auto* session) { return session->read(key); }, m_holder); +} + +template +void session_variant::write(const shared_bytes& key, const shared_bytes& value) { + std::visit([&](auto* session) { return session->write(key, value); }, m_holder); +} + +template +bool session_variant::contains(const shared_bytes& key) { + return std::visit([&](auto* session) { return session->contains(key); }, m_holder); +} + +template +void session_variant::erase(const shared_bytes& key) { + std::visit([&](auto* session) { return session->erase(key); }, m_holder); +} + +template +void session_variant::clear() { + std::visit([&](auto* session) { return session->clear(); }, m_holder); +} + +template +template +const std::pair>, std::unordered_set> +session_variant::read(const Iterable& keys) { + return std::visit([&](auto* session) { return session->read(keys); }, m_holder); +} + +template +template +void session_variant::write(const Iterable& key_values) { + std::visit([&](auto* session) { return session->write(key_values); }, m_holder); +} + +template +template +void session_variant::erase(const Iterable& keys) { + std::visit([&](auto* session) { return session->erase(keys); }, m_holder); +} + +template +template +void session_variant::write_to(Other_data_store& ds, const Iterable& keys) { + std::visit([&](auto* session) { return session->write_to(ds, keys); }, m_holder); +} + +template +template +void session_variant::read_from(Other_data_store& ds, const Iterable& keys) { + std::visit([&](auto* session) { return session->read_from(ds, keys); }, m_holder); +} + +template +typename session_variant::iterator session_variant::find(const shared_bytes& key) { + return std::visit([&](auto* session) { return session_variant::iterator{ session->find(key) }; }, m_holder); +} + +template +typename session_variant::iterator session_variant::begin() { + return std::visit([&](auto* session) { return session_variant::iterator{ session->begin() }; }, m_holder); +} + +template +typename session_variant::iterator session_variant::end() { + return std::visit([&](auto* session) { return session_variant::iterator{ session->end() }; }, m_holder); +} + +template +typename session_variant::iterator session_variant::lower_bound(const shared_bytes& key) { + return std::visit([&](auto* session) { return session_variant::iterator{ session->lower_bound(key) }; }, + m_holder); +} + +template +template +template +session_variant::session_variant_iterator::session_variant_iterator(U iterator) + : m_holder{ std::move(iterator) } {} + +template +template +typename session_variant::template session_variant_iterator& +session_variant::session_variant_iterator::operator++() { + std::visit([&](auto& it) { ++it; }, m_holder); + return *this; +} + +template +template +typename session_variant::template session_variant_iterator& +session_variant::session_variant_iterator::operator--() { + std::visit([&](auto& it) { --it; }, m_holder); + return *this; +} + +template +template +typename session_variant::template session_variant_iterator::value_type +session_variant::session_variant_iterator::operator*() const { + return std::visit([&](auto& it) { return *it; }, m_holder); +} + +template +template +bool session_variant::session_variant_iterator::operator==( + const session_variant_iterator& other) const { + return m_holder == other.m_holder; +} + +template +template +bool session_variant::session_variant_iterator::operator!=( + const session_variant_iterator& other) const { + return m_holder != other.m_holder; +} + +template +template +bool session_variant::session_variant_iterator::deleted() const { + return std::visit([&](auto& it) { return it.deleted(); }, m_holder); +} + +template +template +const shared_bytes session_variant::session_variant_iterator::key() const { + return std::visit([&](auto& it) { return it.key(); }, m_holder); +} + +} // namespace eosio::session diff --git a/libraries/chain_kv/include/b1/session/shared_bytes.hpp b/libraries/chain_kv/include/b1/session/shared_bytes.hpp new file mode 100644 index 00000000000..89fa3c67398 --- /dev/null +++ b/libraries/chain_kv/include/b1/session/shared_bytes.hpp @@ -0,0 +1,565 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include + +namespace eosio::session { + +class shared_bytes; + +template +shared_bytes make_shared_bytes(std::array&& data); + +/// \brief A structure that represents a pointer and its length. +class shared_bytes { + public: + using underlying_type_t = char; + + template + friend struct std::hash; + + template + friend shared_bytes make_shared_bytes(std::array&& data); + + /// \brief Random access iterator on a shared_bytes instance. + template + class shared_bytes_iterator { + public: + friend shared_bytes; + + using difference_type = typename Iterator_traits::difference_type; + using value_type = typename Iterator_traits::value_type; + using pointer = typename Iterator_traits::pointer; + using reference = typename Iterator_traits::reference; + using iterator_category = typename Iterator_traits::iterator_category; + + shared_bytes_iterator() = default; + shared_bytes_iterator(const shared_bytes_iterator& other) = default; + shared_bytes_iterator(shared_bytes_iterator&& other) = default; + + /// \brief Constructor. + /// \param buffer A pointer to the beginning of the buffer that this iterator is iterating over. + /// \param begin The starting point of the range within the buffer the iterator iterates over, inclusive. + /// \param end The ending point of the range within the buffer the iterator iterates over, inclusive. + /// \param index The starting index within the range to start iterating. + /// \remarks -1 for the index indicates the end iterator. + shared_bytes_iterator(char* buffer, int64_t begin, int64_t end, int64_t index); + + shared_bytes_iterator& operator=(const shared_bytes_iterator& other) = default; + shared_bytes_iterator& operator=(shared_bytes_iterator&& other) = default; + shared_bytes_iterator& operator++(); + shared_bytes_iterator& operator--(); + value_type operator*() const; + value_type operator->() const; + reference operator*(); + reference operator->(); + bool operator==(const shared_bytes_iterator& other) const; + bool operator!=(const shared_bytes_iterator& other) const; + shared_bytes_iterator& operator+=(int64_t offset); + shared_bytes_iterator& operator-=(int64_t offset); + shared_bytes_iterator operator+(int64_t offset); + shared_bytes_iterator operator-(int64_t offset); + difference_type operator-(const shared_bytes_iterator& other); + reference operator[](size_t index); + value_type operator[](size_t index) const; + + private: + underlying_type_t* m_buffer{ nullptr }; + int64_t m_end{ 0 }; + int64_t m_begin{ 0 }; + int64_t m_index{ -1 }; + }; + + struct iterator_traits { + using difference_type = int64_t; + using value_type = underlying_type_t; + using pointer = value_type*; + using reference = value_type&; + using iterator_category = std::random_access_iterator_tag; + }; + using iterator = shared_bytes_iterator; + + public: + /// \brief Default Constructor. + /// \remarks Constructs a shared_bytes with a nullptr and no size. + shared_bytes() = default; + + /// \brief Constructs a shared_bytes from an array and size. + /// \param data A pointer to the beginning of an array. + /// \param size The length of the array. + /// \remarks The data is copied into this instance. Memory is aligned on uint64_t boundary. + /// \tparam T The data type of the items in the array. + template + shared_bytes(const T* data, size_t size); + + /// \brief Constructs a shared_bytes with a buffer of the given size. + /// \remarks Memory is aligned on uint64_t boundary. + shared_bytes(size_t size); + + shared_bytes(const shared_bytes& b) = default; + shared_bytes(shared_bytes&& b) = default; + ~shared_bytes() = default; + + shared_bytes& operator=(const shared_bytes& b) = default; + shared_bytes& operator=(shared_bytes&& b) = default; + + bool operator==(const shared_bytes& other) const; + bool operator!=(const shared_bytes& other) const; + bool operator<(const shared_bytes& other) const; + bool operator<=(const shared_bytes& other) const; + bool operator>(const shared_bytes& other) const; + bool operator>=(const shared_bytes& other) const; + + underlying_type_t& operator[](size_t index); + underlying_type_t operator[](size_t index) const; + + /// \brief Not operator. + /// \return True if the underlying pointer is null or the size is 0, false otherwise. + bool operator!() const; + + /// \brief bool conversion operator. + /// \return True if the underlying pointer is not null or the size is greater than 0, false otherwise. + operator bool() const; + + /// \brief Returns a new shared_bytes instances that represents the next largest value in lexicographical ordering. + /// \remarks This is accomplished by adding one to last value in this shared_bytes instance and carrying over the + /// value + /// to the next byte (from back to front) until no carry over is encountered. + shared_bytes next() const; + size_t size() const; + + /// \brief Returns the size of the buffer when aligned on a the size of uint64_t. + size_t aligned_size() const; + + bool empty() const; + char* data(); + const char* const data() const; + + iterator begin() const; + iterator end() const; + + static shared_bytes from_hex_string(const std::string& str); + static shared_bytes truncate_key(const shared_bytes &key); + + private: + size_t m_size{ 0 }; + size_t m_offset{ 0 }; + std::shared_ptr m_data; +}; + +namespace details { + /// \brief Adjusts the size to be aligned to the given byte_size value. + /// \tparam byte_size The number of bytes of the alignment. + template + inline size_t aligned_size(size_t size) { + return (size + (byte_size - 1)) & ~(byte_size - 1); + } + + /// \brief Compares two aligned buffers, lexicographically. + /// \tparam byte_size The number of bytes of the alignment. + /// \param left A pointer to the beginning of the buffer. + /// \param left_size The unaligned size of the left buffer. + /// \param right A pointer to the beginning of the buffer. + /// \param right_size The unalinged size of the right buffer. + /// \return + /// - A value less than 0 if left is less than right. + /// - A value greater than 0 if left is greater than right. + /// - A value of 0 if both buffers are equal. + /// \remarks This comparison works by walking over the buffers, taking each 8 bytes and type punning that to a + /// uint64_t, swapping the bytes of those integers and then performing integer comparison. + template + inline int64_t aligned_compare(const char* left, int64_t left_size, const char* right, int64_t right_size) { + auto iterations = std::min(aligned_size(left_size), aligned_size(right_size)) / byte_size; + for (size_t i = 0; i < iterations; ++i) { + auto offset = i * byte_size; + auto left_value = uint64_t{}; + auto right_value = uint64_t{}; + std::memcpy(&left_value, left + offset, byte_size); + std::memcpy(&right_value, right + offset, byte_size); + // swizzle the bytes before performing the comparison. + left_value = BOOST_ENDIAN_INTRINSIC_BYTE_SWAP_8(left_value); + right_value = BOOST_ENDIAN_INTRINSIC_BYTE_SWAP_8(right_value); + if (left_value == right_value) { + continue; + } + return left_value < right_value ? -1 : 1; + } + return left_size - right_size; + } +} // namespace details + +/// \brief Constructs a new shared_bytes instance from an array of StringView instances. +/// \tparam StringView A type that contains a data() and size() method. +/// \tparam N The size of the array containing StringView instances. +/// \param data An array of StringView instances. +/// \returns A new shared_bytes instance which is the concatenation of all the StringView instances. +template +shared_bytes make_shared_bytes(std::array&& data) { + auto result = shared_bytes{}; + + const std::size_t length = + std::accumulate(data.begin(), data.end(), 0, [](std::size_t a, const StringView& b) { return a + b.size(); }); + + if (length == 0) { + return result; + } + + result.m_size = length; + result.m_offset = details::aligned_size(length) - length; + result.m_data = std::shared_ptr{ + new shared_bytes::underlying_type_t[result.m_size + result.m_offset], + std::default_delete() + }; + char* chunk_ptr = result.m_data.get(); + for (const auto& view : data) { + const char* const view_ptr = view.data(); + if (!view_ptr || !view.size()) { + continue; + } + std::memcpy(chunk_ptr, view_ptr, view.size()); + chunk_ptr += view.size(); + } + std::memset(chunk_ptr, 0, result.m_offset); + + return result; +} + +template +shared_bytes::shared_bytes(const T* data, size_t size) + : m_size{ size * sizeof(T) }, m_offset{ eosio::session::details::aligned_size(m_size) - m_size }, m_data{ [&]() { + if (!data || size == 0) { + return std::shared_ptr{}; + } + + // Make sure to instantiate a buffer that is aligned to the size of a uint64_t. + auto actual_size = m_size + m_offset; + auto result = std::shared_ptr{ new underlying_type_t[actual_size], + std::default_delete() }; + auto* buffer = result.get(); + std::memcpy(buffer, reinterpret_cast(data), m_size); + // Pad with zeros at the end. + std::memset(buffer + m_size, 0, m_offset); + return result; + }() } {} + +inline shared_bytes::shared_bytes(size_t size) + : m_size{ size }, m_offset{ eosio::session::details::aligned_size(m_size) - m_size }, m_data{ [&]() { + if (size == 0) { + return std::shared_ptr{}; + } + + // Make sure to instantiate a buffer that is aligned to the size of a uint64_t. + auto actual_size = m_size + m_offset; + auto result = std::shared_ptr{ new underlying_type_t[actual_size], + std::default_delete() }; + std::memset(result.get(), 0, actual_size); + return result; + }() } {} + +inline shared_bytes shared_bytes::next() const { + auto buffer = std::vector{ std::begin(*this), std::end(*this) }; + + while (!buffer.empty()) { + if (++buffer.back()) { + break; + } + buffer.pop_back(); + } + + EOS_ASSERT(!buffer.empty(), eosio::chain::chain_exception, "shared_bytes::next() result buffer is empty"); + return eosio::session::shared_bytes(buffer.data(), buffer.size()); +} + +inline size_t shared_bytes::size() const { return m_size; } +inline size_t shared_bytes::aligned_size() const { return eosio::session::details::aligned_size(m_size); } +inline char* shared_bytes::data() { return m_data.get(); } +inline const char* const shared_bytes::data() const { return m_data.get(); } + +inline bool shared_bytes::empty() const { return m_size == 0; } + +inline bool shared_bytes::operator==(const shared_bytes& other) const { + if (m_data.get() == other.m_data.get()) { + return true; + } + if (size() != other.size()) { + return false; + } + return details::aligned_compare(m_data.get(), m_size, other.m_data.get(), other.m_size) == 0; +} + +inline bool shared_bytes::operator!=(const shared_bytes& other) const { + if (m_data.get() == other.m_data.get()) { + return false; + } + if (size() != other.size()) { + return true; + } + return details::aligned_compare(m_data.get(), m_size, other.m_data.get(), other.m_size) != 0; +} + +inline bool shared_bytes::operator<(const shared_bytes& other) const { + if (m_data.get() == other.m_data.get()) { + return false; + } + return details::aligned_compare(m_data.get(), m_size, other.m_data.get(), other.m_size) < 0; +} + +inline bool shared_bytes::operator<=(const shared_bytes& other) const { + if (m_data.get() == other.m_data.get()) { + return true; + } + return details::aligned_compare(m_data.get(), m_size, other.m_data.get(), other.m_size) <= 0; +} + +inline bool shared_bytes::operator>(const shared_bytes& other) const { + if (m_data.get() == other.m_data.get()) { + return false; + } + return details::aligned_compare(m_data.get(), m_size, other.m_data.get(), other.m_size) > 0; +} + +inline bool shared_bytes::operator>=(const shared_bytes& other) const { + if (m_data.get() == other.m_data.get()) { + return true; + } + return details::aligned_compare(m_data.get(), m_size, other.m_data.get(), other.m_size) >= 0; +} + +inline bool shared_bytes::operator!() const { return *this == shared_bytes{}; } + +inline shared_bytes::operator bool() const { return *this != shared_bytes{}; } + +inline shared_bytes::underlying_type_t& shared_bytes::operator[](size_t index) { return m_data.get()[index]; } + +inline shared_bytes::underlying_type_t shared_bytes::operator[](size_t index) const { return m_data.get()[index]; } + +inline shared_bytes::iterator shared_bytes::begin() const { + return iterator{ m_data.get(), 0, static_cast(m_size) - 1, m_size == 0 ? -1 : 0 }; +} + +inline shared_bytes::iterator shared_bytes::end() const { + return iterator{ m_data.get(), 0, static_cast(m_size) - 1, -1 }; +} + +inline shared_bytes shared_bytes::truncate_key(const shared_bytes &key) { + EOS_ASSERT(!key.empty(), eosio::chain::chain_exception, "chain_plugin::truncate_key() invalid key parameter: empty"); + + return shared_bytes(key.data(), key.size() - 1); +} + +inline std::ostream& operator<<(std::ostream& os, const shared_bytes& bytes) { + for (const auto c : bytes) { + std::cout << std::hex << std::setfill('0') << std::setw(2) << std::uppercase << (0xFF & static_cast(c)); + } + return os; +} + +template +inline Stream& operator<<(Stream& ds, const shared_bytes& b) { + fc::raw::pack( ds, b.size() ); + ds.write(b.data(), b.size()); + return ds; +} + +template +inline Stream& operator>>(Stream& ds, shared_bytes& b) { + std::size_t sz; + fc::raw::unpack( ds, sz ); + shared_bytes tmp = {sz}; + ds.read(tmp.data(), tmp.size()); + b = tmp; + return ds; +} + +inline shared_bytes shared_bytes::from_hex_string(const std::string& str) { + if (str.empty()) { + return shared_bytes{}; + } + + auto bytes = std::vector{}; + bytes.reserve(str.size() / 2); + for (size_t i = 0; i < str.size(); i += 2) { + std::string hex = str.substr(i, 2); + int8_t result = std::stoi(hex, 0, 16); + char c{ 0 }; + std::memcpy(&c, &result, 1); + bytes.push_back(c); + } + + return shared_bytes{ bytes.data(), bytes.size() }; +} + +template +shared_bytes::shared_bytes_iterator::shared_bytes_iterator(char* buffer, int64_t begin, int64_t end, + int64_t index) + : m_buffer{ buffer }, m_end{ end }, m_begin{ begin }, m_index{ index } {} + +template +shared_bytes::shared_bytes_iterator& +shared_bytes::shared_bytes_iterator::operator++() { + if (m_index < m_end) { + ++m_index; + } else if (m_index == m_end) { + m_index = -1; + } + return *this; +} + +template +shared_bytes::shared_bytes_iterator& +shared_bytes::shared_bytes_iterator::operator--() { + if (m_index > m_begin) { + --m_index; + } + return *this; +} + +template +typename shared_bytes::shared_bytes_iterator::value_type +shared_bytes::shared_bytes_iterator::operator*() const { + return m_buffer[m_index]; +} + +template +typename shared_bytes::shared_bytes_iterator::value_type +shared_bytes::shared_bytes_iterator::operator->() const { + return m_buffer[m_index]; +} + +template +typename shared_bytes::shared_bytes_iterator::reference +shared_bytes::shared_bytes_iterator::operator*() { + return m_buffer[m_index]; +} + +template +typename shared_bytes::shared_bytes_iterator::reference +shared_bytes::shared_bytes_iterator::operator->() { + return m_buffer[m_index]; +} + +template +bool shared_bytes::shared_bytes_iterator::operator==(const shared_bytes_iterator& other) const { + return m_buffer == other.m_buffer && m_end == other.m_end && m_begin == other.m_begin && m_index == other.m_index; +} + +template +bool shared_bytes::shared_bytes_iterator::operator!=(const shared_bytes_iterator& other) const { + return !(*this == other); +} + +template +shared_bytes::shared_bytes_iterator& +shared_bytes::shared_bytes_iterator::operator+=(int64_t offset) { + if (m_index + offset <= m_end) { + m_index += offset; + } else { + m_index = -1; + } + return *this; +} + +template +shared_bytes::shared_bytes_iterator& +shared_bytes::shared_bytes_iterator::operator-=(int64_t offset) { + if (m_index - offset >= m_begin) { + m_index -= offset; + } else { + m_index = m_begin; + } + return *this; +} + +template +shared_bytes::shared_bytes_iterator +shared_bytes::shared_bytes_iterator::operator+(int64_t offset) { + auto new_iterator = *this; + if (m_index + offset <= m_end) { + new_iterator.m_index += offset; + } else { + new_iterator.m_index = -1; + } + return new_iterator; +} + +template +shared_bytes::shared_bytes_iterator +shared_bytes::shared_bytes_iterator::operator-(int64_t offset) { + auto new_iterator = *this; + if (m_index - offset >= m_begin) { + new_iterator.m_index -= offset; + } else { + new_iterator.m_index = m_begin; + } + return new_iterator; +} + +template +typename shared_bytes::shared_bytes_iterator::difference_type +shared_bytes::shared_bytes_iterator::operator-(const shared_bytes_iterator& other) { + auto left_index = m_index == -1 ? m_end + 1 : m_index; + auto right_index = other.m_index == -1 ? other.m_end + 1 : other.m_index; + return std::abs(left_index - right_index); +} + +template +typename shared_bytes::shared_bytes_iterator::reference +shared_bytes::shared_bytes_iterator::operator[](size_t index) { + return m_buffer[index]; +} + +template +typename shared_bytes::shared_bytes_iterator::value_type +shared_bytes::shared_bytes_iterator::operator[](size_t index) const { + return m_buffer[index]; +} + +} // namespace eosio::session + +namespace std { + +template <> +struct less { + bool operator()(const eosio::session::shared_bytes& lhs, const eosio::session::shared_bytes& rhs) const { + return lhs < rhs; + }; +}; + +template <> +struct greater { + bool operator()(const eosio::session::shared_bytes& lhs, const eosio::session::shared_bytes& rhs) const { + return lhs > rhs; + }; +}; + +template <> +struct hash { + size_t operator()(const eosio::session::shared_bytes& b) const { + if (b.size() == 0) { + return 0; + } + return std::hash{}({ b.m_data.get() + b.m_offset, b.m_size }); + } +}; + +template <> +struct equal_to { + bool operator()(const eosio::session::shared_bytes& lhs, const eosio::session::shared_bytes& rhs) const { + return lhs == rhs; + } +}; + +} // namespace std diff --git a/libraries/chain_kv/include/b1/session/undo_stack.hpp b/libraries/chain_kv/include/b1/session/undo_stack.hpp new file mode 100644 index 00000000000..63f49aaecec --- /dev/null +++ b/libraries/chain_kv/include/b1/session/undo_stack.hpp @@ -0,0 +1,324 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace eosio::session { + constexpr uint32_t undo_stack_magic_number = 0x30510ABC; + constexpr uint32_t undo_stack_min_supported_version = 1; + constexpr uint32_t undo_stack_max_supported_version = 1; + constexpr auto undo_stack_filename = "undo_stack.dat"; + +/// \brief Represents a container of pending sessions to be committed. +template +class undo_stack { + public: + using root_type = Session; + using session_type = session; + using variant_type = session_variant; + using const_variant_type = session_variant; + + /// \brief Constructor. + /// \param head The session that the changes are merged into when commit is called. + undo_stack(Session& head, const fc::path& datadir = {}); + undo_stack(const undo_stack&) = delete; + undo_stack(undo_stack&&) = default; + ~undo_stack(); + + undo_stack& operator=(const undo_stack&) = delete; + undo_stack& operator=(undo_stack&&) = default; + + /// \brief Adds a new session to the top of the stack. + void push(); + + /// \brief Merges the changes of the top session into the session below it in the stack. + void squash(); + + /// \brief Pops the top session off the stack and discards the changes in that session. + void undo(); + + /// \brief Commits the sessions at the bottom of the stack up to and including the provided revision. + /// \param revision The revision number to commit up to. + /// \remarks Each time a session is push onto the stack, a revision is assigned to it. + void commit(int64_t revision); + + bool empty() const; + size_t size() const; + + /// \brief The starting revision number of the stack. + int64_t revision() const; + + /// \brief Sets the starting revision number of the stack. + /// \remarks This can only be set when the stack is empty and it can't be set + /// to value lower than the current revision. + void revision(int64_t revision); + + /// \brief Returns the head session (the session at the top of the stack. + variant_type top(); + + /// \brief Returns the head session (the session at the top of the stack. + const_variant_type top() const; + + /// \brief Returns the session at the bottom of the stack. + /// \remarks This is the next session to be committed. + variant_type bottom(); + + /// \brief Returns the session at the bottom of the stack. + /// \remarks This is the next session to be committed. + const_variant_type bottom() const; + + void open(); + void close(); + + private: + int64_t m_revision{ 0 }; + Session* m_head; + std::deque m_sessions; // Need a deque so pointers don't become invalidated. The session holds a + // pointer to the parent internally. + fc::path m_datadir; +}; + +template +undo_stack::undo_stack(Session& head, const fc::path& datadir) : m_head{ &head }, m_datadir{ datadir } { + open(); +} + +template +undo_stack::~undo_stack() { + close(); +} + +template +void undo_stack::push() { + if (m_sessions.empty()) { + m_sessions.emplace_back(*m_head); + } else { + m_sessions.emplace_back(m_sessions.back(), nullptr); + } + ++m_revision; +} + +template +void undo_stack::squash() { + if (m_sessions.empty()) { + return; + } + m_sessions.back().commit(); + m_sessions.back().detach(); + m_sessions.pop_back(); + --m_revision; +} + +template +void undo_stack::undo() { + if (m_sessions.empty()) { + return; + } + m_sessions.back().detach(); + m_sessions.pop_back(); + --m_revision; +} + +template +void undo_stack::commit(int64_t revision) { + if (m_sessions.empty()) { + return; + } + + revision = std::min(revision, m_revision); + auto initial_revision = static_cast(m_revision - m_sessions.size() + 1); + if (initial_revision > revision) { + return; + } + + const auto start_index = revision - initial_revision; + + for (int64_t i = start_index; i >= 0; --i) { m_sessions[i].commit(); } + m_sessions.erase(std::begin(m_sessions), std::begin(m_sessions) + start_index + 1); + if (!m_sessions.empty()) { + m_sessions.front().attach(*m_head); + } +} + +template +bool undo_stack::empty() const { + return m_sessions.empty(); +} + +template +size_t undo_stack::size() const { + return m_sessions.size(); +} + +template +int64_t undo_stack::revision() const { + return m_revision; +} + +template +void undo_stack::revision(int64_t revision) { + if (!empty()) { + return; + } + + if (revision <= m_revision) { + return; + } + + m_revision = revision; +} + +template +typename undo_stack::variant_type undo_stack::top() { + if (!m_sessions.empty()) { + auto& back = m_sessions.back(); + return { back, nullptr }; + } + return { *m_head, nullptr }; +} + +template +typename undo_stack::const_variant_type undo_stack::top() const { + if (!m_sessions.empty()) { + auto& back = m_sessions.back(); + return { back, nullptr }; + } + return { *m_head, nullptr }; +} + +template +typename undo_stack::variant_type undo_stack::bottom() { + if (!m_sessions.empty()) { + auto& front = m_sessions.front(); + return { front, nullptr }; + } + return { *m_head, nullptr }; +} + +template +typename undo_stack::const_variant_type undo_stack::bottom() const { + if (!m_sessions.empty()) { + auto& front = m_sessions.front(); + return { front, nullptr }; + } + return { *m_head, nullptr }; +} + +template +void undo_stack::open() { + if (m_datadir.empty()) + return; + + if (!fc::is_directory(m_datadir)) + fc::create_directories(m_datadir); + + auto undo_stack_dat = m_datadir / undo_stack_filename; + if( fc::exists( undo_stack_dat ) ) { + try { + std::string content; + fc::read_file_contents( undo_stack_dat, content ); + + fc::datastream ds( content.data(), content.size() ); + + // validate totem + uint32_t totem = 0; + fc::raw::unpack( ds, totem ); + EOS_ASSERT( totem == undo_stack_magic_number, eosio::chain::chain_exception, + "Undo stack data file '${filename}' has unexpected magic number: ${actual_totem}. Expected ${expected_totem}", + ("filename", undo_stack_dat.generic_string()) + ("actual_totem", totem) + ("expected_totem", undo_stack_magic_number) + ); + + // validate version + uint32_t version = 0; + fc::raw::unpack( ds, version ); + EOS_ASSERT( version >= undo_stack_min_supported_version && version <= undo_stack_max_supported_version, + eosio::chain::chain_exception, + "Unsupported version of Undo stack data file '${filename}'. " + "Undo stack data version is ${version} while code supports version(s) [${min},${max}]", + ("filename", undo_stack_dat.generic_string()) + ("version", version) + ("min", undo_stack_min_supported_version) + ("max", undo_stack_max_supported_version) + ); + + int64_t rev; fc::raw::unpack( ds, rev ); + + size_t num_sessions; fc::raw::unpack( ds, num_sessions ); + for( size_t i = 0; i < num_sessions; ++i ) { + push(); + auto& session = m_sessions.back(); + + size_t num_updated_keys; fc::raw::unpack( ds, num_updated_keys ); + for( size_t j = 0; j < num_updated_keys; ++j ) { + shared_bytes key; ds >> key; + shared_bytes value; ds >> value; + session.write(key, value); + } + + size_t num_deleted_keys; fc::raw::unpack( ds, num_deleted_keys ); + for( size_t j = 0; j < num_deleted_keys; ++j ) { + shared_bytes key; ds >> key; + session.erase(key); + } + } + m_revision = rev; // restore head revision + } FC_CAPTURE_AND_RETHROW( (undo_stack_dat) ) + + fc::remove( undo_stack_dat ); + } +} + +template +void undo_stack::close() { + if (m_datadir.empty()) + return; + + auto undo_stack_dat = m_datadir / undo_stack_filename; + + std::ofstream out( undo_stack_dat.generic_string().c_str(), std::ios::out | std::ios::binary | std::ofstream::trunc ); + fc::raw::pack( out, undo_stack_magic_number ); + fc::raw::pack( out, undo_stack_max_supported_version ); + + fc::raw::pack( out, revision() ); + fc::raw::pack( out, size() ); // number of sessions + + while ( !m_sessions.empty() ) { + auto& session = m_sessions.front(); + auto updated_keys = session.updated_keys(); + fc::raw::pack( out, updated_keys.size() ); // number of updated keys + + for (const auto& key: updated_keys) { + auto value = session.read(key); + + if ( value ) { + out << key; + out << *value; + } else { + fc::remove( undo_stack_dat ); // May not be used by next startup + elog( "Did not find value for ${k}", ("k", key.data() ) ); + return; // Do not assert as we are during shutdown + } + } + + auto deleted_keys = session.deleted_keys(); + fc::raw::pack( out, deleted_keys.size() ); // number of deleted keys + + for (const auto& key: deleted_keys) { + fc::raw::pack( out, key ); + } + + session.detach(); + m_sessions.pop_front(); + } +} +} // namespace eosio::session diff --git a/libraries/chain_kv/unit_tests/CMakeLists.txt b/libraries/chain_kv/unit_tests/CMakeLists.txt new file mode 100644 index 00000000000..984ef7f78cb --- /dev/null +++ b/libraries/chain_kv/unit_tests/CMakeLists.txt @@ -0,0 +1,24 @@ +include(ExternalProject) + +file(GLOB UNIT_TESTS "*.cpp") # find all unit test suites + +add_executable(chain-kv-tests ${UNIT_TESTS}) +target_link_libraries(chain-kv-tests chain_kv eosio_chain) +target_include_directories(chain-kv-tests PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../../chain/include") + +enable_testing() + +### MARK TEST SUITES FOR EXECUTION ### +foreach(TEST_SUITE ${UNIT_TESTS}) # create an independent target for each test suite + execute_process(COMMAND bash -c "grep -E 'BOOST_AUTO_TEST_SUITE\\s*[(]' '${TEST_SUITE}' | grep -vE '//.*BOOST_AUTO_TEST_SUITE\\s*[(]' | cut -d ')' -f 1 | cut -d '(' -f 2" OUTPUT_VARIABLE SUITE_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) # get the test suite name from the *.cpp file + if (NOT "" STREQUAL "${SUITE_NAME}") # ignore empty lines + execute_process(COMMAND bash -c "echo ${SUITE_NAME} | sed -e 's/s$//' | sed -e 's/_test$//'" OUTPUT_VARIABLE TRIMMED_SUITE_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) # trim "_test" or "_tests" from the end of ${SUITE_NAME} + add_test(NAME chain_kv_${TRIMMED_SUITE_NAME}_unit_test COMMAND chain-kv-tests --run_test=${SUITE_NAME} --report_level=detailed --color_output --catch_system_errors=no) + # build list of tests to run during coverage testing + if(ctest_tests) + string(APPEND ctest_tests "|") + endif() + string(APPEND ctest_tests chain_kv_${TRIMMED_SUITE_NAME}_unit_test) + endif() +endforeach(TEST_SUITE) +set(ctest_tests "'${ctest_tests}' -j8") diff --git a/libraries/chain_kv/unit_tests/chain_kv_tests.hpp b/libraries/chain_kv/unit_tests/chain_kv_tests.hpp new file mode 100644 index 00000000000..ed0cb8f2c37 --- /dev/null +++ b/libraries/chain_kv/unit_tests/chain_kv_tests.hpp @@ -0,0 +1,114 @@ +#include +#include + +namespace chain_kv = b1::chain_kv; + +struct kv_values { + std::vector> values; + + friend bool operator==(const kv_values& a, const kv_values& b) { return a.values == b.values; } +}; + +template +S& operator<<(S& s, const kv_values& v) { + s << "{\n"; + for (auto& kv : v.values) { + s << " {{"; + bool first = true; + for (auto b : kv.first) { + if (!first) + s << ", "; + first = false; + char x[10]; + sprintf(x, "0x%02x", (uint8_t)b); + s << x; + } + s << "}, {"; + first = true; + for (auto b : kv.second) { + if (!first) + s << ", "; + first = false; + char x[10]; + sprintf(x, "0x%02x", (uint8_t)b); + s << x; + } + s << "}},\n"; + } + s << "}"; + return s; +} + +#define KV_REQUIRE_EXCEPTION(expr, msg) \ + BOOST_REQUIRE_EXCEPTION(expr, chain_kv::exception, [](auto& e) { \ + if (!strcmp(e.what(), msg)) \ + return true; \ + std::cerr << "expected: '" \ + << "\033[33m" << msg << "\033[0m'\ngot: '\033[33m" << e.what() << "'\033[0m\n"; \ + return false; \ + }); + +inline kv_values get_all(chain_kv::database& db, const chain_kv::bytes& prefix) { + kv_values result; + std::unique_ptr rocks_it{ db.rdb->NewIterator(rocksdb::ReadOptions()) }; + rocks_it->Seek(chain_kv::to_slice(prefix)); + while (rocks_it->Valid()) { + auto k = rocks_it->key(); + if (k.size() < prefix.size() || memcmp(k.data(), prefix.data(), prefix.size())) + break; + auto v = rocks_it->value(); + result.values.push_back({ chain_kv::to_bytes(k), chain_kv::to_bytes(v) }); + rocks_it->Next(); + } + if (!rocks_it->status().IsNotFound()) + chain_kv::check(rocks_it->status(), "iterate: "); + return result; +} + +inline kv_values get_values(chain_kv::write_session& session, const std::vector& keys) { + kv_values result; + for (auto& key : keys) { + chain_kv::bytes value; + if (auto value = session.get(chain_kv::bytes{ key })) + result.values.push_back({ key, *value }); + } + return result; +} + +inline kv_values get_matching(chain_kv::view& view, uint64_t contract, const chain_kv::bytes& prefix = {}) { + kv_values result; + chain_kv::view::iterator it{ view, contract, chain_kv::to_slice(prefix) }; + ++it; + while (!it.is_end()) { + auto kv = it.get_kv(); + if (!kv) + throw chain_kv::exception("iterator read failure"); + result.values.push_back({ chain_kv::to_bytes(kv->key), chain_kv::to_bytes(kv->value) }); + ++it; + } + return result; +} + +inline kv_values get_matching2(chain_kv::view& view, uint64_t contract, const chain_kv::bytes& prefix = {}) { + kv_values result; + chain_kv::view::iterator it{ view, contract, chain_kv::to_slice(prefix) }; + --it; + while (!it.is_end()) { + auto kv = it.get_kv(); + if (!kv) + throw chain_kv::exception("iterator read failure"); + result.values.push_back({ chain_kv::to_bytes(kv->key), chain_kv::to_bytes(kv->value) }); + --it; + } + std::reverse(result.values.begin(), result.values.end()); + return result; +} + +inline kv_values get_it(const chain_kv::view::iterator& it) { + kv_values result; + auto kv = it.get_kv(); + if (!kv) + throw chain_kv::exception("iterator read failure"); + result.values.push_back({ chain_kv::to_bytes(kv->key), chain_kv::to_bytes(kv->value) }); + return result; +} diff --git a/libraries/chain_kv/unit_tests/data_store_tests.hpp b/libraries/chain_kv/unit_tests/data_store_tests.hpp new file mode 100644 index 00000000000..e96cd4c80f4 --- /dev/null +++ b/libraries/chain_kv/unit_tests/data_store_tests.hpp @@ -0,0 +1,643 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +namespace eosio::session_tests { + +inline std::shared_ptr make_rocks_db(const std::string& name = "/tmp/testdb") { + rocksdb::DestroyDB(name.c_str(), rocksdb::Options{}); + + rocksdb::DB* cache_ptr{ nullptr }; + auto cache = std::shared_ptr{}; + + auto options = rocksdb::Options{}; + options.create_if_missing = true; + options.level_compaction_dynamic_level_bytes = true; + options.bytes_per_sync = 1048576; + options.OptimizeLevelStyleCompaction(256ull << 20); + + auto status = rocksdb::DB::Open(options, name.c_str(), &cache_ptr); + cache.reset(cache_ptr); + + return cache; +} + +static const std::unordered_map char_key_values{ + { "a", "123456789" }, + { "b", "abcdefghi" }, + { "c", "987654321" }, + { "d", "ABCDEFGHI" }, + { "e", "HELLO WORLD" }, + { "f", "Hello World" }, + { "aa", "Foo" }, + { "bb", "Bar" }, + { "cc", "FooBar" }, + { "dd", "Fizz" }, + { "ee", "Buzz" }, + { "ff", "FizzBuzz" }, + { "aaa", "qwerty" }, + { "bbb", "QWERTY" }, + { "ccc", "10101010101010" }, + { "ddd", "00000000000000" }, + { "eee", "01010101010101" }, + { "fff", "11111111111111" }, + { "aaaaa", "000000001111111" }, + { "bbbbb", "111111110000000" }, + { "ccccc", "1" }, + { "ddddd", "2" }, + { "eeeee", "3" }, + { "fffff", "5" }, + { "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" }, + { "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, + { "cccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "dddddddddddddddddddddddddddddddddddddddddddddddddddddd" }, + { "dddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "cccccccccccccccccccccccccccccccccccccccccccccccccccccc" }, + { "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "ffffffffffffffffffffffffffffffffffffffffffffffffffffff" }, + { "ffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "ffffffffffffffffffffffffffffffffffffffffffffffffffffff" }, +}; + +static const std::vector> char_batch_values{ + { eosio::session::shared_bytes("hello0", 6), eosio::session::shared_bytes("world0", 6) }, + { eosio::session::shared_bytes("hello1", 6), eosio::session::shared_bytes("world1", 6) }, + { eosio::session::shared_bytes("hello2", 6), eosio::session::shared_bytes("world2", 6) }, + { eosio::session::shared_bytes("hello3", 6), eosio::session::shared_bytes("world3", 6) }, + { eosio::session::shared_bytes("hello4", 6), eosio::session::shared_bytes("world4", 6) }, + { eosio::session::shared_bytes("hello5", 6), eosio::session::shared_bytes("world5", 6) }, + { eosio::session::shared_bytes("hello6", 6), eosio::session::shared_bytes("world6", 6) }, + { eosio::session::shared_bytes("hello7", 6), eosio::session::shared_bytes("world7", 6) }, + { eosio::session::shared_bytes("hello8", 6), eosio::session::shared_bytes("world8", 6) }, + { eosio::session::shared_bytes("hello9", 6), eosio::session::shared_bytes("world9", 6) }, +}; + +static const std::unordered_map int_key_values{ + { 1, 1 }, { 3, 2 }, { 5, 3 }, { 7, 4 }, { 9, 5 }, { 11, 6 }, { 13, 7 }, { 15, 8 }, + { 14, 9 }, { 12, 10 }, { 10, 11 }, { 8, 12 }, { 6, 13 }, { 4, 14 }, { 2, 15 }, +}; + +static const std::vector int_keys{ + 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, +}; + +static const std::vector int_values{ + 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000, +}; + +static const std::vector> int_batch_values{ + { eosio::session::shared_bytes(&int_keys[0], 1), eosio::session::shared_bytes(&int_values[0], 1) }, + { eosio::session::shared_bytes(&int_keys[1], 1), eosio::session::shared_bytes(&int_values[1], 1) }, + { eosio::session::shared_bytes(&int_keys[2], 1), eosio::session::shared_bytes(&int_values[2], 1) }, + { eosio::session::shared_bytes(&int_keys[3], 1), eosio::session::shared_bytes(&int_values[3], 1) }, + { eosio::session::shared_bytes(&int_keys[4], 1), eosio::session::shared_bytes(&int_values[4], 1) }, + { eosio::session::shared_bytes(&int_keys[5], 1), eosio::session::shared_bytes(&int_values[5], 1) }, + { eosio::session::shared_bytes(&int_keys[6], 1), eosio::session::shared_bytes(&int_values[6], 1) }, + { eosio::session::shared_bytes(&int_keys[7], 1), eosio::session::shared_bytes(&int_values[7], 1) }, + { eosio::session::shared_bytes(&int_keys[8], 1), eosio::session::shared_bytes(&int_values[8], 1) }, + { eosio::session::shared_bytes(&int_keys[9], 1), eosio::session::shared_bytes(&int_values[9], 1) }, +}; + +struct string_t {}; +struct int_t {}; + +template +void make_data_store(T& ds, const std::unordered_map& kvs, string_t) { + for (const auto& kv : kvs) { + ds.write(eosio::session::shared_bytes(kv.first.c_str(), kv.first.size()), + eosio::session::shared_bytes(kv.second.c_str(), kv.second.size())); + } +} + +template +void make_data_store(T& ds, const std::unordered_map& kvs, int_t) { + for (const auto& kv : kvs) { + ds.write(eosio::session::shared_bytes(&kv.first, 1), eosio::session::shared_bytes(&kv.second, 1)); + } +} + +template +void verify_equal(T& ds, const std::unordered_map& container, string_t) { + auto verify_key_value = [&](auto kv) { + auto key = std::string{ std::begin(kv.first), std::end(kv.first) }; + auto it = container.find(key); + BOOST_REQUIRE(it != std::end(container)); + auto buffer = std::vector{ std::begin(*kv.second), + std::end(*kv.second) }; + BOOST_REQUIRE(std::memcmp(it->second.c_str(), buffer.data(), it->second.size()) == 0); + }; + + for (const auto& kv : ds) { verify_key_value(kv); } + + auto begin = std::begin(ds); + auto end = std::end(ds); + auto current = end; + --current; + auto count = size_t{ 0 }; + while (true) { + verify_key_value(*current); + ++count; + if (current == begin) { + break; + } + --current; + } + BOOST_REQUIRE(count == container.size()); + + for (const auto& it : container) { + auto key = eosio::session::shared_bytes(it.first.c_str(), it.first.size()); + auto value = ds.read(key); + BOOST_REQUIRE(ds.contains(key) == true); + BOOST_REQUIRE(value.has_value()); + auto buffer = + std::vector{ std::begin(*value), std::end(*value) }; + BOOST_REQUIRE(std::memcmp(it.second.c_str(), buffer.data(), it.second.size()) == 0); + } +} + +template +void verify_equal(eosio::session::session& ds, const std::unordered_map& container, string_t) { + auto verify_key_value = [&](auto kv) { + auto key = std::string{ std::begin(kv.first), std::end(kv.first) }; + auto it = container.find(key); + BOOST_REQUIRE(it != std::end(container)); + auto buffer = std::vector{ std::begin(*kv.second), + std::end(*kv.second) }; + BOOST_REQUIRE(std::memcmp(it->second.c_str(), buffer.data(), it->second.size()) == 0); + }; + + // the iterator is a session is circular. So we need to bail out when we circle around to the beginning. + auto begin = std::begin(ds); + auto kv_it = std::begin(ds); + auto count = size_t{ 0 }; + do { + if (kv_it != std::end(ds)) { + verify_key_value(*kv_it); + ++count; + } + ++kv_it; + } while (kv_it != begin); + BOOST_REQUIRE(count == container.size()); + + kv_it = std::end(ds); + --kv_it; + count = 0; + while (true) { + verify_key_value(*kv_it); + ++count; + if (kv_it == begin) { + break; + } + --kv_it; + } + BOOST_REQUIRE(count == container.size()); + + for (const auto& it : container) { + auto key = eosio::session::shared_bytes(it.first.c_str(), it.first.size()); + auto value = ds.read(key); + BOOST_REQUIRE(ds.contains(key) == true); + BOOST_REQUIRE(value.has_value()); + auto buffer = + std::vector{ std::begin(*value), std::end(*value) }; + BOOST_REQUIRE(std::memcmp(it.second.c_str(), buffer.data(), it.second.size()) == 0); + } +} + +template +void verify_equal(T& ds, const std::unordered_map& container, int_t) { + auto verify_key_value = [&](auto kv) { + auto buffer = + std::vector{ std::begin(kv.first), std::end(kv.first) }; + auto it = container.find(*reinterpret_cast(buffer.data())); + BOOST_REQUIRE(it != std::end(container)); + buffer = std::vector{ std::begin(*kv.second), + std::end(*kv.second) }; + BOOST_REQUIRE(std::memcmp(reinterpret_cast(&it->second), buffer.data(), sizeof(Value)) == 0); + }; + + for (const auto& kv : ds) { verify_key_value(kv); } + + auto begin = std::begin(ds); + auto end = std::end(ds); + auto current = end; + --current; + auto count = size_t{ 0 }; + while (true) { + verify_key_value(*current); + ++count; + if (current == begin) { + break; + } + --current; + } + BOOST_REQUIRE(count == container.size()); + + for (const auto& it : container) { + auto key = eosio::session::shared_bytes(&it.first, 1); + auto value = ds.read(key); + BOOST_REQUIRE(value.has_value()); + BOOST_REQUIRE(ds.contains(key) == true); + auto buffer = + std::vector{ std::begin(*value), std::end(*value) }; + BOOST_REQUIRE(std::memcmp(reinterpret_cast(&it.second), buffer.data(), sizeof(Value)) == 0); + } +} + +template +void verify_equal(eosio::session::session& ds, const std::unordered_map& container, int_t) { + auto verify_key_value = [&](auto kv) { + auto buffer = + std::vector{ std::begin(kv.first), std::end(kv.first) }; + auto it = container.find(*reinterpret_cast(buffer.data())); + BOOST_REQUIRE(it != std::end(container)); + buffer = std::vector{ std::begin(*kv.second), + std::end(*kv.second) }; + BOOST_REQUIRE(std::memcmp(reinterpret_cast(&it->second), buffer.data(), sizeof(Value)) == 0); + }; + + // the iterator is a session is circular. So we need to bail out when we circle around to the beginning. + auto begin = std::begin(ds); + auto kv_it = std::begin(ds); + auto count = size_t{ 0 }; + do { + if (kv_it != std::end(ds)) { + verify_key_value(*kv_it); + ++count; + } + ++kv_it; + } while (kv_it != begin); + BOOST_REQUIRE(count == container.size()); + + kv_it = std::end(ds); + --kv_it; + count = 0; + while (true) { + verify_key_value(*kv_it); + ++count; + if (kv_it == begin) { + break; + } + --kv_it; + } + BOOST_REQUIRE(count == container.size()); + + for (const auto& it : container) { + auto key = eosio::session::shared_bytes(&it.first, 1); + auto value = ds.read(key); + BOOST_REQUIRE(value.has_value()); + BOOST_REQUIRE(ds.contains(key) == true); + auto buffer = + std::vector{ std::begin(*value), std::end(*value) }; + BOOST_REQUIRE(std::memcmp(reinterpret_cast(&it.second), buffer.data(), sizeof(Value)) == 0); + } +} + +template +void verify_iterators(T& ds, string_t) { + BOOST_REQUIRE(ds.find(eosio::session::shared_bytes("g", 1)) == std::end(ds)); + BOOST_REQUIRE(ds.find(eosio::session::shared_bytes("a", 1)) != std::end(ds)); + BOOST_REQUIRE(*ds.find(eosio::session::shared_bytes("a", 1)) == + std::pair(eosio::session::shared_bytes("a", 1), std::optional{ + eosio::session::shared_bytes("123456789", 9) })); + BOOST_REQUIRE(*std::begin(ds) == + std::pair(eosio::session::shared_bytes("a", 1), std::optional{ + eosio::session::shared_bytes("123456789", 9) })); + BOOST_REQUIRE(std::begin(ds) != std::end(ds)); + BOOST_REQUIRE(*ds.lower_bound(eosio::session::shared_bytes("fffff", 5)) == + std::pair(eosio::session::shared_bytes("fffff", 5), + std::optional{ eosio::session::shared_bytes("5", 1) })); +} + +template +void verify_iterators(T& ds, int_t) { + auto search_key = int32_t{ 16 }; + BOOST_REQUIRE(ds.find(eosio::session::shared_bytes(&search_key, 1)) == std::end(ds)); + search_key = 15; + auto search_value = 8; + BOOST_REQUIRE(ds.find(eosio::session::shared_bytes(&search_key, 1)) != std::end(ds)); + BOOST_REQUIRE( + *ds.find(eosio::session::shared_bytes(&search_key, 1)) == + std::pair(eosio::session::shared_bytes(&search_key, 1), + std::optional{ eosio::session::shared_bytes(&search_value, 1) })); + search_key = 1; + search_value = 1; + BOOST_REQUIRE(*std::begin(ds) == std::pair(eosio::session::shared_bytes(&search_key, 1), + std::optional{ + eosio::session::shared_bytes(&search_value, 1) })); + BOOST_REQUIRE(std::begin(ds) != std::end(ds)); + search_key = 14; + search_value = 9; + auto result_key = int32_t{ 14 }; + auto result_value = int32_t{ 9 }; + BOOST_REQUIRE( + *ds.lower_bound(eosio::session::shared_bytes(&search_key, 1)) == + std::pair(eosio::session::shared_bytes(&result_key, 1), + std::optional{ eosio::session::shared_bytes(&result_value, 1) })); +} + +template +void verify_key_order(T& ds) { + auto begin_key = eosio::session::shared_bytes{}; + auto current_key = eosio::session::shared_bytes{}; + auto compare = std::less{}; + for (const auto& kv : ds) { + if (!current_key) { + current_key = kv.first; + begin_key = kv.first; + continue; + } + + if (current_key == begin_key) { + // We've wrapped around + break; + } + + BOOST_REQUIRE(compare(current_key, kv.first) == true); + current_key = kv.first; + } +} + +template +void verify_session_key_order(T& ds) { + auto current_key = eosio::session::shared_bytes{}; + auto compare = std::less{}; + + // the iterator is a session is circular. So we need to bail out when we circle around to the beginning. + auto begin = std::begin(ds); + auto kv_it = std::begin(ds); + do { + if (kv_it == std::end(ds)) { + ++kv_it; + continue; + } + + auto kv = *kv_it; + if (!current_key) { + current_key = kv.first; + ++kv_it; + continue; + } + + BOOST_REQUIRE(compare(current_key, kv.first) == true); + current_key = kv.first; + ++kv_it; + } while (kv_it != begin); +} + +template +void verify_rwd(T& ds, const eosio::session::shared_bytes& key, const eosio::session::shared_bytes& value) { + BOOST_REQUIRE(!ds.read(key).has_value()); + BOOST_REQUIRE(ds.contains(key) == false); + + ds.write(key, value); + BOOST_REQUIRE(ds.read(key) == value); + BOOST_REQUIRE(ds.contains(key) == true); + + ds.erase(key); + BOOST_REQUIRE(!ds.read(key).has_value()); + BOOST_REQUIRE(ds.contains(key) == false); +} + +template +void verify_rwd_batch(T& ds, const Iterable& kvs) { + auto keys = std::vector{}; + for (const auto& kv : kvs) { keys.emplace_back(kv.first); } + + auto [read_batch1, not_found1] = ds.read(keys); + BOOST_REQUIRE(read_batch1.empty() == true); + for (const auto& kv : kvs) { + BOOST_REQUIRE(!ds.read(kv.first).has_value()); + BOOST_REQUIRE(ds.contains(kv.first) == false); + BOOST_REQUIRE(not_found1.find(kv.first) != std::end(not_found1)); + } + + ds.write(kvs); + auto [read_batch2, not_found2] = ds.read(keys); + BOOST_REQUIRE(read_batch2.empty() == false); + for (const auto& kv : kvs) { + BOOST_REQUIRE(ds.read(kv.first).has_value()); + BOOST_REQUIRE(ds.contains(kv.first) == true); + BOOST_REQUIRE(not_found2.find(kv.first) == std::end(not_found2)); + } + + ds.erase(keys); + auto [read_batch3, not_found3] = ds.read(keys); + BOOST_REQUIRE(read_batch3.empty() == true); + for (const auto& kv : kvs) { + BOOST_REQUIRE(!ds.read(kv.first).has_value()); + BOOST_REQUIRE(ds.contains(kv.first) == false); + BOOST_REQUIRE(not_found3.find(kv.first) != std::end(not_found3)); + } +} + +template +void verify_read_from_datastore(T& ds, U& other_ds) { + auto compare_ds = [](auto& left, auto& right) { + // The data stores are equal if all the key_values in left are in right + // and all the key_values in right are in left. + for (const auto& kv : left) { + BOOST_REQUIRE(right.contains(kv.first) == true); + BOOST_REQUIRE(right.read(kv.first) == kv.second); + } + + for (const auto& kv : right) { + BOOST_REQUIRE(left.contains(kv.first) == true); + BOOST_REQUIRE(left.read(kv.first) == kv.second); + } + }; + + auto keys = std::vector{}; + for (const auto& kv : ds) { keys.emplace_back(kv.first); } + + other_ds.read_from(ds, keys); + compare_ds(other_ds, ds); +} + +template +void verify_read_from_datastore(eosio::session::session& ds, + eosio::session::session& other_ds) { + auto compare_ds = [](auto& left, auto& right) { + // The data stores are equal if all the key_values in left are in right + // and all the key_values in right are in left. + // the iterator is a session is circular. So we need to bail out when we circle around to the beginning. + auto begin1 = std::begin(left); + auto kv_it1 = std::begin(left); + do { + if (kv_it1 != std::end(left)) { + auto kv = *kv_it1; + BOOST_REQUIRE(right.contains(kv.first) == true); + BOOST_REQUIRE(right.read(kv.first) == kv.second); + } + ++kv_it1; + } while (kv_it1 != begin1); + + auto begin2 = std::begin(right); + auto kv_it2 = std::begin(right); + do { + if (kv_it2 != std::end(right)) { + auto kv = *kv_it2; + BOOST_REQUIRE(left.contains(kv.first) == true); + BOOST_REQUIRE(left.read(kv.first) == kv.second); + } + ++kv_it2; + } while (kv_it2 != begin2); + }; + + auto keys = std::vector{}; + auto begin = std::begin(ds); + auto kv_it = std::begin(ds); + do { + if (kv_it != std::end(ds)) { + auto kv = *kv_it; + keys.emplace_back(kv.first); + } + ++kv_it; + } while (kv_it != begin); + + other_ds.read_from(ds, keys); + compare_ds(other_ds, ds); +} + +template +void verify_write_to_datastore(T& ds, U& other_ds) { + auto compare_ds = [](auto& left, auto& right) { + // The data stores are equal if all the key_values in left are in right + // and all the key_values in right are in left. + for (const auto& kv : left) { + BOOST_REQUIRE(right.contains(kv.first) == true); + BOOST_REQUIRE(right.read(kv.first) == kv.second); + } + + for (const auto& kv : right) { + BOOST_REQUIRE(left.contains(kv.first) == true); + BOOST_REQUIRE(left.read(kv.first) == kv.second); + } + }; + + auto keys = std::vector{}; + for (const auto& kv : ds) { keys.emplace_back(kv.first); } + + ds.write_to(other_ds, keys); + compare_ds(other_ds, ds); +} + +template +void verify_write_to_datastore(eosio::session::session& ds, eosio::session::session& other_ds) { + auto compare_ds = [](auto& left, auto& right) { + // The data stores are equal if all the key_values in left are in right + // and all the key_values in right are in left. + // the iterator is a session is circular. So we need to bail out when we circle around to the beginning. + auto begin1 = std::begin(left); + auto kv_it1 = std::begin(left); + do { + if (kv_it1 != std::end(left)) { + auto kv = *kv_it1; + BOOST_REQUIRE(right.contains(kv.first) == true); + BOOST_REQUIRE(right.read(kv.first) == kv.second); + } + ++kv_it1; + } while (kv_it1 != begin1); + + auto begin2 = std::begin(right); + auto kv_it2 = std::begin(right); + do { + if (kv_it2 != std::end(right)) { + auto kv = *kv_it2; + BOOST_REQUIRE(left.contains(kv.first) == true); + BOOST_REQUIRE(left.read(kv.first) == kv.second); + } + ++kv_it2; + } while (kv_it2 != begin2); + }; + + auto keys = std::vector{}; + auto begin = std::begin(ds); + auto kv_it = std::begin(ds); + do { + if (kv_it != std::end(ds)) { + auto kv = *kv_it; + keys.emplace_back(kv.first); + } + ++kv_it; + } while (kv_it != begin); + + ds.write_to(other_ds, keys); + compare_ds(other_ds, ds); +} + +inline eosio::session::session make_session(const std::string& name = "/tmp/testdb") { + auto rocksdb = make_rocks_db(name); + return eosio::session::make_session(std::move(rocksdb), 16); +} + +template +void verify(const Data_store& ds, const Container& kvs) { + for (auto kv : ds) { + auto buffer = + std::vector{ std::begin(kv.first), std::end(kv.first) }; + auto current_key = *reinterpret_cast(buffer.data()); + buffer = std::vector{ std::begin(*kv.second), + std::end(*kv.second) }; + auto current_value = *reinterpret_cast(buffer.data()); + + auto it = kvs.find(current_key); + BOOST_REQUIRE(it != std::end(kvs)); + BOOST_REQUIRE(it->first == current_key); + BOOST_REQUIRE(it->second == current_value); + } + + for (auto kv : kvs) { + auto key = eosio::session::shared_bytes(&kv.first, 1); + auto value = eosio::session::shared_bytes(&kv.second, 1); + auto result_value = ds.read(key); + BOOST_REQUIRE(result_value.value()); + BOOST_REQUIRE(value == result_value); + } +}; + +template +void write(Data_store& ds, const Container& kvs) { + for (auto kv : kvs) { + ds.write(eosio::session::shared_bytes(&kv.first, 1), eosio::session::shared_bytes(&kv.second, 1)); + } +}; + +inline std::unordered_map generate_kvs(size_t size) { + std::random_device random_device; + std::mt19937 generator{ random_device() }; + std::uniform_int_distribution distribution{ 0, std::numeric_limits::max() }; + + auto container = std::unordered_map{}; + for (size_t i = 0; i < size; ++i) { container.emplace(distribution(generator), distribution(generator)); } + return container; +}; + +inline std::unordered_map +collapse(const std::vector>& kvs_list) { + if (kvs_list.empty()) { + return std::unordered_map{}; + } + + auto merged = kvs_list[0]; + for (size_t i = 1; i < kvs_list.size(); ++i) { + auto& list = kvs_list[i]; + + for (auto kv : list) { merged.insert_or_assign(kv.first, kv.second); } + } + return merged; +}; + +} // namespace eosio::session_tests diff --git a/libraries/chain_kv/unit_tests/main.cpp b/libraries/chain_kv/unit_tests/main.cpp new file mode 100644 index 00000000000..99a2beae010 --- /dev/null +++ b/libraries/chain_kv/unit_tests/main.cpp @@ -0,0 +1,36 @@ +#include +#include +#include +#include +#include + +void translate_fc_exception(const fc::exception& e) { + std::cerr << "\033[33m" << e.to_detail_string() << "\033[0m" << std::endl; + BOOST_TEST_FAIL("Caught Unexpected Exception"); +} + +boost::unit_test::test_suite* init_unit_test_suite(int argc, char* argv[]) { + // Turn off logging if no --verbose parameter is not added + // To have verbose enabled, call "unit_tests/unit_test -- --verbose" + bool is_verbose = false; + std::string verbose_arg = "--verbose"; + for (int i = 0; i < argc; i++) { + if (verbose_arg == argv[i]) { + is_verbose = true; + break; + } + } + if (is_verbose) { + fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug); + } else { + fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::off); + } + + // Register fc::exception translator + boost::unit_test::unit_test_monitor.register_exception_translator(&translate_fc_exception); + + std::srand(time(NULL)); + std::cout << "Random number generator seeded to " << time(NULL) << std::endl; + + return nullptr; +} diff --git a/libraries/chain_kv/unit_tests/rocks_session_tests.cpp b/libraries/chain_kv/unit_tests/rocks_session_tests.cpp new file mode 100644 index 00000000000..2604a342955 --- /dev/null +++ b/libraries/chain_kv/unit_tests/rocks_session_tests.cpp @@ -0,0 +1,113 @@ +#include "data_store_tests.hpp" +#include + +using namespace eosio::session; +using namespace eosio::session_tests; + +BOOST_AUTO_TEST_SUITE(rocks_session_tests) + +BOOST_AUTO_TEST_CASE(rocks_session_create_test) { + { + auto datastore1 = eosio::session_tests::make_session("/tmp/rocks1"); + make_data_store(datastore1, char_key_values, string_t{}); + verify_equal(datastore1, char_key_values, string_t{}); + } + + { + auto datastore2 = eosio::session_tests::make_session("/tmp/rocks2"); + make_data_store(datastore2, int_key_values, int_t{}); + verify_equal(datastore2, int_key_values, int_t{}); + } +} + +BOOST_AUTO_TEST_CASE(rocks_session_rwd_test) { + { + auto datastore1 = eosio::session_tests::make_session("/tmp/rocks3"); + make_data_store(datastore1, char_key_values, string_t{}); + for (const auto& kv : char_batch_values) { verify_rwd(datastore1, kv.first, kv.second); } + } + + { + auto datastore2 = eosio::session_tests::make_session("/tmp/rocks4"); + make_data_store(datastore2, int_key_values, int_t{}); + for (const auto& kv : int_batch_values) { verify_rwd(datastore2, kv.first, kv.second); } + } +} + +BOOST_AUTO_TEST_CASE(rocks_session_rwd_batch_test) { + { + auto datastore1 = eosio::session_tests::make_session("/tmp/rocks5"); + make_data_store(datastore1, char_key_values, string_t{}); + verify_rwd_batch(datastore1, char_batch_values); + } + + { + auto datastore2 = eosio::session_tests::make_session("/tmp/rocks6"); + make_data_store(datastore2, int_key_values, int_t{}); + verify_rwd_batch(datastore2, int_batch_values); + } +} + +BOOST_AUTO_TEST_CASE(rocks_session_rw_ds_test) { + { + auto datastore1 = eosio::session_tests::make_session("/tmp/rocks7"); + auto datastore2 = eosio::session_tests::make_session("/tmp/rocks8"); + make_data_store(datastore1, char_key_values, string_t{}); + verify_read_from_datastore(datastore1, datastore2); + } + + { + auto datastore3 = eosio::session_tests::make_session("/tmp/rocks18"); + auto datastore4 = eosio::session_tests::make_session("/tmp/rocks9"); + make_data_store(datastore3, int_key_values, int_t{}); + verify_write_to_datastore(datastore3, datastore4); + } +} + +BOOST_AUTO_TEST_CASE(rocks_session_iterator_test) { + { + auto datastore1 = eosio::session_tests::make_session("/tmp/rocks10"); + make_data_store(datastore1, char_key_values, string_t{}); + verify_iterators(datastore1, string_t{}); + } + { + auto datastore2 = eosio::session_tests::make_session("/tmp/rocks11"); + make_data_store(datastore2, char_key_values, string_t{}); + verify_iterators(datastore2, string_t{}); + } + { + auto datastore3 = eosio::session_tests::make_session("/tmp/rocks12"); + make_data_store(datastore3, int_key_values, int_t{}); + verify_iterators(datastore3, int_t{}); + } + { + auto datastore4 = eosio::session_tests::make_session("/tmp/rocks13"); + make_data_store(datastore4, int_key_values, int_t{}); + verify_iterators(datastore4, int_t{}); + } +} + +BOOST_AUTO_TEST_CASE(rocks_session_iterator_key_order_test) { + { + auto datastore1 = eosio::session_tests::make_session("/tmp/rocks14"); + make_data_store(datastore1, char_key_values, string_t{}); + verify_key_order(datastore1); + } + { + auto datastore2 = eosio::session_tests::make_session("/tmp/rocks15"); + make_data_store(datastore2, char_key_values, string_t{}); + verify_key_order(datastore2); + } + { + auto datastore3 = eosio::session_tests::make_session("/tmp/rocks16"); + make_data_store(datastore3, int_key_values, int_t{}); + verify_key_order(datastore3); + } + { + auto datastore4 = eosio::session_tests::make_session("/tmp/rocks17"); + make_data_store(datastore4, int_key_values, int_t{}); + verify_key_order(datastore4); + } +} + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/libraries/chain_kv/unit_tests/session_tests.cpp b/libraries/chain_kv/unit_tests/session_tests.cpp new file mode 100644 index 00000000000..b32f1fe5512 --- /dev/null +++ b/libraries/chain_kv/unit_tests/session_tests.cpp @@ -0,0 +1,463 @@ +#include "data_store_tests.hpp" +#include +#include + +using namespace eosio::session; +using namespace eosio::session_tests; + +namespace eosio::session_tests { + +void perform_session_level_test(const std::string& dbpath, bool always_undo = false) { + auto kvs_list = std::vector>{}; + auto ordered_list = std::vector>{}; + + auto root_session = eosio::session_tests::make_session(dbpath); + using session_type = eosio::session::session; + kvs_list.emplace_back(generate_kvs(50)); + ordered_list.emplace_back(std::begin(kvs_list.back()), std::end(kvs_list.back())); + write(root_session, kvs_list.back()); + verify_equal(root_session, kvs_list.back(), int_t{}); + verify_session_key_order(root_session); + + for (size_t i = 0; i < 3; ++i) { + auto block_session = session_type(root_session); + kvs_list.emplace_back(generate_kvs(50)); + ordered_list.emplace_back(std::begin(kvs_list.back()), std::end(kvs_list.back())); + write(block_session, kvs_list.back()); + verify_equal(block_session, collapse(kvs_list), int_t{}); + verify_session_key_order(block_session); + + for (size_t j = 0; j < 3; ++j) { + auto transaction = session_type(block_session, nullptr); + kvs_list.emplace_back(generate_kvs(50)); + ordered_list.emplace_back(std::begin(kvs_list.back()), std::end(kvs_list.back())); + write(transaction, kvs_list.back()); + verify_equal(transaction, collapse(kvs_list), int_t{}); + verify_session_key_order(transaction); + + if (j % 2 == 1 || always_undo) { + transaction.undo(); + kvs_list.pop_back(); + } else { + transaction.commit(); + transaction.detach(); + } + + // Verify contents of block session + verify_equal(block_session, collapse(kvs_list), int_t{}); + verify_session_key_order(block_session); + } + + // Verify contents of block session + verify_equal(block_session, collapse(kvs_list), int_t{}); + verify_session_key_order(block_session); + + if (i % 2 == 1 || always_undo) { + block_session.undo(); + kvs_list.pop_back(); + + if (!always_undo) { + // Pop the last two transaction kvs as well. + kvs_list.pop_back(); + kvs_list.pop_back(); + } + } else { + block_session.commit(); + block_session.detach(); + } + + // Verify contents of root session + verify_equal(root_session, collapse(kvs_list), int_t{}); + verify_session_key_order(root_session); + } + // Verify contents of root session + verify_equal(root_session, collapse(kvs_list), int_t{}); + verify_session_key_order(root_session); +} + +} // namespace eosio::session_tests + +BOOST_AUTO_TEST_SUITE(session_tests) + +BOOST_AUTO_TEST_CASE(session_create_test) { + { + auto session1 = eosio::session_tests::make_session("/tmp/session18"); + make_data_store(session1, char_key_values, string_t{}); + verify_equal(session1, char_key_values, string_t{}); + } + { + auto session2 = eosio::session_tests::make_session("/tmp/session19"); + make_data_store(session2, int_key_values, int_t{}); + verify_equal(session2, int_key_values, int_t{}); + } +} + +BOOST_AUTO_TEST_CASE(session_rwd_test) { + { + auto session1 = eosio::session_tests::make_session("/tmp/session20"); + make_data_store(session1, char_key_values, string_t{}); + for (const auto& kv : char_batch_values) { verify_rwd(session1, kv.first, kv.second); } + } + { + auto session2 = eosio::session_tests::make_session("/tmp/session21"); + make_data_store(session2, int_key_values, int_t{}); + for (const auto& kv : int_batch_values) { verify_rwd(session2, kv.first, kv.second); } + } +} + +BOOST_AUTO_TEST_CASE(session_rwd_batch_test) { + { + auto session1 = eosio::session_tests::make_session("/tmp/session1"); + make_data_store(session1, char_key_values, string_t{}); + verify_rwd_batch(session1, char_batch_values); + } + { + auto session2 = eosio::session_tests::make_session("/tmp/session2"); + make_data_store(session2, int_key_values, int_t{}); + verify_rwd_batch(session2, int_batch_values); + } +} + +BOOST_AUTO_TEST_CASE(session_rw_ds_test) { + { + auto session1 = eosio::session_tests::make_session("/tmp/session3"); + auto session2 = eosio::session_tests::make_session("/tmp/session4"); + make_data_store(session1, char_key_values, string_t{}); + verify_read_from_datastore(session1, session2); + } + { + auto session3 = eosio::session_tests::make_session("/tmp/session5"); + auto session4 = eosio::session_tests::make_session("/tmp/session6"); + make_data_store(session3, int_key_values, int_t{}); + verify_write_to_datastore(session3, session4); + } +} + +BOOST_AUTO_TEST_CASE(session_iterator_test) { + { + auto session1 = eosio::session_tests::make_session("/tmp/session7"); + make_data_store(session1, char_key_values, string_t{}); + verify_iterators(session1, string_t{}); + } + { + auto session2 = eosio::session_tests::make_session("/tmp/session8"); + make_data_store(session2, char_key_values, string_t{}); + verify_iterators(session2, string_t{}); + } + { + auto session3 = eosio::session_tests::make_session("/tmp/session9"); + make_data_store(session3, int_key_values, int_t{}); + verify_iterators(session3, int_t{}); + } + { + auto session4 = eosio::session_tests::make_session("/tmp/session10"); + make_data_store(session4, int_key_values, int_t{}); + verify_iterators(session4, int_t{}); + } +} + +BOOST_AUTO_TEST_CASE(session_iterator_key_order_test) { + { + auto session1 = eosio::session_tests::make_session("/tmp/session11"); + make_data_store(session1, char_key_values, string_t{}); + verify_session_key_order(session1); + } + { + auto session2 = eosio::session_tests::make_session("/tmp/session12"); + make_data_store(session2, char_key_values, string_t{}); + verify_session_key_order(session2); + } + { + auto session3 = eosio::session_tests::make_session("/tmp/session13"); + make_data_store(session3, int_key_values, int_t{}); + verify_session_key_order(session3); + } + { + auto session4 = eosio::session_tests::make_session("/tmp/session14"); + make_data_store(session4, int_key_values, int_t{}); + verify_session_key_order(session4); + } +} + +BOOST_AUTO_TEST_CASE(session_level_test_undo_sometimes) { + eosio::session_tests::perform_session_level_test("/tmp/session22"); +} + +BOOST_AUTO_TEST_CASE(session_level_test_undo_always) { + eosio::session_tests::perform_session_level_test("/tmp/session23", true); +} + +BOOST_AUTO_TEST_CASE(session_level_test_attach_detach) { + size_t key_count = 10; + auto root_session = eosio::session_tests::make_session("/tmp/session15"); + using session_type = eosio::session::session; + auto root_session_kvs = generate_kvs(key_count); + write(root_session, root_session_kvs); + verify_equal(root_session, root_session_kvs, int_t{}); + verify_session_key_order(root_session); + + auto block_sessions = std::vector{}; + auto block_session_kvs = std::vector>{}; + for (size_t i = 0; i < 3; ++i) { + block_sessions.emplace_back(session_type(root_session)); + block_session_kvs.emplace_back(generate_kvs(key_count)); + write(block_sessions.back(), block_session_kvs.back()); + verify_equal(block_sessions.back(), collapse({ root_session_kvs, block_session_kvs.back() }), int_t{}); + verify_session_key_order(block_sessions.back()); + block_sessions.back().detach(); + } + + // Root session should not have changed. + verify_equal(root_session, root_session_kvs, int_t{}); + + auto transaction_sessions = std::vector{}; + auto transaction_session_kvs = std::vector>{}; + for (size_t i = 0; i < 3; ++i) { + auto& block_session = block_sessions[i]; + + block_session.attach(root_session); + auto& kvs = block_session_kvs[i]; + + for (size_t j = 0; j < 3; ++j) { + transaction_sessions.emplace_back(session_type(block_session, nullptr)); + transaction_session_kvs.emplace_back(generate_kvs(key_count)); + write(transaction_sessions.back(), transaction_session_kvs.back()); + verify_equal(transaction_sessions.back(), collapse({ root_session_kvs, kvs, transaction_session_kvs.back() }), + int_t{}); + verify_session_key_order(transaction_sessions.back()); + transaction_sessions.back().detach(); + } + + // Block session should not have changed. + verify_equal(block_session, collapse({ root_session_kvs, kvs }), int_t{}); + block_session.detach(); + } + + // Root session should not have changed. + verify_equal(root_session, root_session_kvs, int_t{}); + + // Attach each block and transaction, attach and commit. + auto session_kvs = std::vector{}; + session_kvs.emplace_back(root_session_kvs); + for (size_t i = 0; i < block_sessions.size(); ++i) { + auto& block_session = block_sessions[i]; + block_session.attach(root_session); + session_kvs.emplace_back(block_session_kvs[i]); + + for (size_t j = 3 * i; j < 3 * i + 3; ++j) { + auto& transaction_session = transaction_sessions[j]; + transaction_session.attach(block_session); + transaction_session.commit(); + session_kvs.emplace_back(transaction_session_kvs[j]); + } + + block_session.commit(); + } + + // Verify contents of root session + verify_equal(root_session, collapse(session_kvs), int_t{}); + verify_session_key_order(root_session); +} + +BOOST_AUTO_TEST_CASE(session_overwrite_key_in_child) { + auto verify_key_value = [](auto& ds, uint16_t key, uint16_t expected_value) { + auto key_ = eosio::session::shared_bytes(&key, 1); + auto value = eosio::session::shared_bytes(&expected_value, 1); + auto value_read = ds.read(key_); + BOOST_REQUIRE(value_read == value); + + auto begin = std::begin(ds); + auto it = std::begin(ds); + auto end = std::end(ds); + do { + if (it == end) { + ++it; + continue; + } + + auto key_value = *it; + if (key_value.first == key_) { + BOOST_REQUIRE(key_value.second == value); + break; + } + ++it; + } while (it != begin); + }; + + auto root_session = eosio::session_tests::make_session("/tmp/session16"); + using session_type = eosio::session::session; + auto root_session_kvs = + std::unordered_map{ { 0, 10 }, { 1, 9 }, { 2, 8 }, { 3, 7 }, { 4, 6 }, { 5, 5 }, + { 6, 4 }, { 7, 3 }, { 8, 2 }, { 9, 1 }, { 10, 0 } }; + write(root_session, root_session_kvs); + verify_equal(root_session, root_session_kvs, int_t{}); + verify_session_key_order(root_session); + + auto block_session = session_type(root_session); + auto block_session_kvs = std::unordered_map{ + { 0, 1000 }, { 1, 1001 }, { 2, 1002 }, { 3, 1003 }, { 4, 1004 }, + }; + write(block_session, block_session_kvs); + // verify_equal(root_session, root_session_kvs, int_t{}); + verify_equal(block_session, collapse({ root_session_kvs, block_session_kvs }), int_t{}); + verify_session_key_order(block_session); + verify_key_value(block_session, 0, 1000); + verify_key_value(block_session, 1, 1001); + verify_key_value(block_session, 2, 1002); + verify_key_value(block_session, 3, 1003); + verify_key_value(block_session, 4, 1004); + + auto transaction_session = session_type(block_session, nullptr); + auto transaction_session_kvs = + std::unordered_map{ { 0, 2000 }, { 1, 2001 }, { 2, 2002 }, { 3, 2003 }, + { 4, 2004 }, { 9, 2005 }, { 10, 2006 } }; + write(transaction_session, transaction_session_kvs); + // verify_equal(root_session, root_session_kvs, int_t{}); + // verify_equal(block_session, collapse({root_session_kvs, block_session_kvs}), int_t{}); + verify_equal(transaction_session, collapse({ root_session_kvs, block_session_kvs, transaction_session_kvs }), + int_t{}); + verify_session_key_order(transaction_session); + verify_key_value(transaction_session, 0, 2000); + verify_key_value(transaction_session, 1, 2001); + verify_key_value(transaction_session, 2, 2002); + verify_key_value(transaction_session, 3, 2003); + verify_key_value(transaction_session, 4, 2004); + verify_key_value(transaction_session, 9, 2005); + verify_key_value(transaction_session, 10, 2006); +} + +BOOST_AUTO_TEST_CASE(session_delete_key_in_child) { + auto verify_keys_deleted = [](auto& ds, const auto& keys) { + for (const uint16_t& key : keys) { + auto key_ = eosio::session::shared_bytes(&key, 1); + BOOST_REQUIRE(!ds.read(key_).has_value()); + BOOST_REQUIRE(ds.find(key_) == std::end(ds)); + BOOST_REQUIRE(ds.contains(key_) == false); + } + + auto begin = std::begin(ds); + auto it = std::begin(ds); + auto end = std::end(ds); + do { + if (it != end) { + auto key = it.key(); + auto buffer = std::vector{ std::begin(key), std::end(key) }; + BOOST_REQUIRE(keys.find(*reinterpret_cast(buffer.data())) == std::end(keys)); + } + ++it; + } while (it != begin); + }; + + auto verify_keys_exist = [](auto& ds, const auto& key_values) { + for (const auto& key_value : key_values) { + auto key = eosio::session::shared_bytes(&key_value.first, 1); + auto value = eosio::session::shared_bytes(&key_value.second, 1); + BOOST_REQUIRE(*ds.read(key) == value); + BOOST_REQUIRE(ds.find(key) != std::end(ds)); + BOOST_REQUIRE(ds.contains(key) == true); + } + + auto found = size_t{ 0 }; + auto begin = std::begin(ds); + auto it = std::begin(ds); + auto end = std::end(ds); + do { + if (it == end) { + ++it; + continue; + } + auto key_value = *it; + auto buffer = + std::vector{ std::begin(key_value.first), std::end(key_value.first) }; + auto key = *reinterpret_cast(buffer.data()); + buffer = std::vector{ std::begin(*key_value.second), + std::end(*key_value.second) }; + auto value = *reinterpret_cast(buffer.data()); + + auto kv_it = key_values.find(key); + if (kv_it != std::end(key_values)) { + BOOST_REQUIRE(value == kv_it->second); + ++found; + } + + ++it; + } while (it != begin); + BOOST_REQUIRE(found == key_values.size()); + }; + + auto delete_key = [](auto& ds, uint16_t key) { + auto key_ = eosio::session::shared_bytes(&key, 1); + ds.erase(key_); + }; + + auto root_session = eosio::session_tests::make_session("/tmp/session17"); + using session_type = eosio::session::session; + auto root_session_kvs = + std::unordered_map{ { 0, 10 }, { 1, 9 }, { 2, 8 }, { 3, 7 }, { 4, 6 }, { 5, 5 }, + { 6, 4 }, { 7, 3 }, { 8, 2 }, { 9, 1 }, { 10, 0 } }; + write(root_session, root_session_kvs); + verify_equal(root_session, root_session_kvs, int_t{}); + verify_session_key_order(root_session); + + auto block_session = session_type(root_session); + delete_key(block_session, 2); + delete_key(block_session, 4); + delete_key(block_session, 6); + verify_keys_deleted(block_session, std::unordered_set{ 2, 4, 6 }); + // verify_equal(root_session, root_session_kvs, int_t{}); + + auto transaction_session = session_type(block_session, nullptr); + auto transaction_session_kvs = std::unordered_map{ { 2, 2003 }, { 4, 2004 } }; + write(transaction_session, transaction_session_kvs); + verify_keys_deleted(transaction_session, std::unordered_set{ 6 }); + verify_keys_exist(transaction_session, std::unordered_map{ { 2, 2003 }, { 4, 2004 } }); + // verify_equal(root_session, root_session_kvs, int_t{}); + + transaction_session.commit(); + verify_keys_deleted(block_session, std::unordered_set{ 6 }); + verify_keys_exist(block_session, std::unordered_map{ { 2, 2003 }, { 4, 2004 } }); + block_session.commit(); + verify_keys_deleted(root_session, std::unordered_set{ 6 }); + verify_keys_exist(root_session, std::unordered_map{ { 2, 2003 }, { 4, 2004 } }); + root_session.commit(); +} + +// BOOST_AUTO_TEST_CASE(session_iteration) { +// using rocks_db_type = rocks_data_store<>; +// using cache_type = cache<>; +// using session_type = session; + +// auto memory_allocator = boost_memory_allocator::make(); +// auto cache_ds = eosio::session::make_cache(memory_allocator); +// auto rocksdb = make_rocks_db("testdb"); +// auto rocks_ds = eosio::session::make_rocks_data_store(std::move(rocksdb), std::move(memory_allocator)); + +// auto root_session = eosio::session::make_session(rocks_ds, cache_ds); +// auto root_session_kvs = generate_kvs(5000); +// write(root_session, root_session_kvs); +// // Commit some data to the database. +// root_session.commit(); + +// auto root_session_kvs_2 = generate_kvs(5000); +// write(root_session, root_session_kvs_2); + +// auto block_session_kvs = generate_kvs(5000); +// auto block_session = eosio::session::make_session(root_session); +// write(block_session, block_session_kvs); + +// auto transaction_session_kvs = generate_kvs(5000); +// auto transaction_session = eosio::session::make_session(block_session); +// write(transaction_session, transaction_session_kvs); + +// auto set = collapse({root_session_kvs, root_session_kvs_2, block_session_kvs, transaction_session_kvs}); + +// // Iterate a few times just for a time measurement. +// for (size_t i = 0; i < 500000; ++i) { +// auto begin = std::begin(transaction_session); +// auto current = std::begin(transaction_session); +// do { +// } while (++current != begin); +// } +// } + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/libraries/chain_kv/unit_tests/session_undo_stack_tests.cpp b/libraries/chain_kv/unit_tests/session_undo_stack_tests.cpp new file mode 100644 index 00000000000..edbc9ebbfed --- /dev/null +++ b/libraries/chain_kv/unit_tests/session_undo_stack_tests.cpp @@ -0,0 +1,96 @@ +#include "data_store_tests.hpp" +#include +#include +#include + +#include + +using namespace eosio::session; +using namespace eosio::session_tests; + +BOOST_AUTO_TEST_SUITE(session_undo_stack_tests) + +BOOST_AUTO_TEST_CASE(undo_stack_test) { + // Push the head session into the undo stack. + auto data_store = eosio::session::make_session(make_rocks_db(), 16); + auto undo = eosio::session::undo_stack(data_store); + auto session_kvs_1 = std::unordered_map{ + { 1, 100 }, { 2, 200 }, { 3, 300 }, { 4, 400 }, { 5, 500 }, + }; + write(data_store, session_kvs_1); + verify_equal(data_store, session_kvs_1, int_t{}); + + auto top = [&]() -> decltype(undo)::session_type& { + return *std::get(undo.top().holder()); + }; + + // Push a new session on the end of the undo stack and write some data to it + undo.push(); + BOOST_REQUIRE(undo.revision() == 1); + + auto session_kvs_2 = std::unordered_map{ + { 6, 600 }, { 7, 700 }, { 8, 800 }, { 9, 900 }, { 10, 1000 }, + }; + write(top(), session_kvs_2); + verify_equal(top(), collapse({ session_kvs_1, session_kvs_2 }), int_t{}); + + // Undo that new session + undo.undo(); + BOOST_REQUIRE(undo.revision() == 0); + BOOST_REQUIRE(undo.empty()); + verify_equal(data_store, session_kvs_1, int_t{}); + + // Push a new session and verify. + undo.push(); + BOOST_REQUIRE(undo.revision() == 1); + write(top(), session_kvs_2); + verify_equal(top(), collapse({ session_kvs_1, session_kvs_2 }), int_t{}); + + auto session_kvs_3 = std::unordered_map{ + { 11, 1100 }, { 12, 1200 }, { 13, 1300 }, { 14, 1400 }, { 15, 1500 }, + }; + undo.push(); + BOOST_REQUIRE(undo.revision() == 2); + write(top(), session_kvs_3); + verify_equal(top(), collapse({ session_kvs_1, session_kvs_2, session_kvs_3 }), int_t{}); + + undo.squash(); + BOOST_REQUIRE(undo.revision() == 1); + verify_equal(top(), collapse({ session_kvs_1, session_kvs_2, session_kvs_3 }), int_t{}); + + auto session_kvs_4 = std::unordered_map{ + { 16, 1600 }, { 17, 1700 }, { 18, 1800 }, { 19, 1900 }, { 20, 2000 }, + }; + undo.push(); + BOOST_REQUIRE(undo.revision() == 2); + write(top(), session_kvs_4); + verify_equal(top(), collapse({ session_kvs_1, session_kvs_2, session_kvs_3, session_kvs_4 }), int_t{}); + + auto session_kvs_5 = std::unordered_map{ + { 21, 2100 }, { 22, 2200 }, { 23, 2300 }, { 24, 2400 }, { 25, 2500 }, + }; + undo.push(); + BOOST_REQUIRE(undo.revision() == 3); + write(top(), session_kvs_5); + verify_equal(top(), collapse({ session_kvs_1, session_kvs_2, session_kvs_3, session_kvs_4, session_kvs_5 }), + int_t{}); + + auto session_kvs_6 = std::unordered_map{ + { 26, 2600 }, { 27, 2700 }, { 28, 2800 }, { 29, 2900 }, { 30, 3000 }, + }; + undo.push(); + BOOST_REQUIRE(undo.revision() == 4); + write(top(), session_kvs_6); + verify_equal(top(), + collapse({ session_kvs_1, session_kvs_2, session_kvs_3, session_kvs_4, session_kvs_5, session_kvs_6 }), + int_t{}); + + // Commit revision 3 and verify that the top session has the correct key values. + undo.commit(3); + BOOST_REQUIRE(undo.revision() == 4); + verify_equal(top(), + collapse({ session_kvs_1, session_kvs_2, session_kvs_3, session_kvs_4, session_kvs_5, session_kvs_6 }), + int_t{}); +} + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/libraries/chain_kv/unit_tests/shared_bytes_tests.cpp b/libraries/chain_kv/unit_tests/shared_bytes_tests.cpp new file mode 100644 index 00000000000..d07c7e70595 --- /dev/null +++ b/libraries/chain_kv/unit_tests/shared_bytes_tests.cpp @@ -0,0 +1,163 @@ +#include +#include + +using namespace eosio::session; + +BOOST_AUTO_TEST_SUITE(shared_bytes_tests) + +BOOST_AUTO_TEST_CASE(make_shared_bytes_test) { + static constexpr auto* char_value = "hello world"; + static const auto char_length = strlen(char_value) - 1; + static constexpr auto int_value = int64_t{ 100000000 }; + + auto b1 = shared_bytes(char_value, char_length); + auto buffer = std::vector{ std::begin(b1), std::end(b1) }; + BOOST_REQUIRE(memcmp(buffer.data(), reinterpret_cast(char_value), char_length) == 0); + BOOST_REQUIRE(b1.size() == char_length); + + auto b2 = shared_bytes(reinterpret_cast(char_value), char_length); + buffer = std::vector{ std::begin(b2), std::end(b2) }; + BOOST_REQUIRE(memcmp(buffer.data(), reinterpret_cast(char_value), char_length) == 0); + BOOST_REQUIRE(b2.size() == char_length); + + auto b3 = shared_bytes(&int_value, 1); + buffer = std::vector{ std::begin(b3), std::end(b3) }; + BOOST_REQUIRE(*(reinterpret_cast(buffer.data())) == int_value); + BOOST_REQUIRE(b3.size() == sizeof(decltype(int_value))); + + auto b4 = shared_bytes(char_value, char_length); + buffer = std::vector{ std::begin(b4), std::end(b4) }; + BOOST_REQUIRE(memcmp(buffer.data(), reinterpret_cast(char_value), char_length) == 0); + BOOST_REQUIRE(b4.size() == char_length); + BOOST_REQUIRE(!b4.empty()); + + auto invalid = shared_bytes(static_cast(nullptr), 0); + BOOST_REQUIRE(std::begin(invalid) == std::end(invalid)); + BOOST_REQUIRE(invalid.size() == 0); + + BOOST_REQUIRE(b1 == b2); + BOOST_REQUIRE(b1 == b4); + BOOST_REQUIRE(invalid == shared_bytes{}); + BOOST_REQUIRE(b1 != b3); + + auto b5 = b1; + buffer = std::vector{ std::begin(b5), std::end(b5) }; + BOOST_REQUIRE(memcmp(buffer.data(), reinterpret_cast(char_value), char_length) == 0); + BOOST_REQUIRE(b5.size() == char_length); + BOOST_REQUIRE(b1 == b5); + + auto b6{ b1 }; + buffer = std::vector{ std::begin(b6), std::end(b6) }; + BOOST_REQUIRE(memcmp(buffer.data(), reinterpret_cast(char_value), char_length) == 0); + BOOST_REQUIRE(b6.size() == char_length); + BOOST_REQUIRE(b1 == b6); + + auto b7 = std::move(b1); + buffer = std::vector{ std::begin(b7), std::end(b7) }; + BOOST_REQUIRE(memcmp(buffer.data(), reinterpret_cast(char_value), char_length) == 0); + BOOST_REQUIRE(b7.size() == char_length); + + auto b8{ std::move(b2) }; + buffer = std::vector{ std::begin(b8), std::end(b8) }; + BOOST_REQUIRE(memcmp(buffer.data(), reinterpret_cast(char_value), char_length) == 0); + BOOST_REQUIRE(b8.size() == char_length); +} + +BOOST_AUTO_TEST_CASE(iterator_test) { + auto s = std::string{ "Hello world foobar" }; + auto bytes = eosio::session::shared_bytes{ s.data(), s.size() }; + auto result = std::string{ std::begin(bytes), std::end(bytes) }; + BOOST_REQUIRE(s == result); +} + +BOOST_AUTO_TEST_CASE(std_copy) { + auto bytes = eosio::session::shared_bytes{ 18 }; + auto parts = std::vector{ "Hello ", "world ", "foobar" }; + auto offset = size_t{ 0 }; + for (const auto& part : parts) { + std::copy(std::begin(part), std::end(part), std::begin(bytes) + offset); + offset += part.size(); + } + auto expected_result = std::string{ "Hello world foobar" }; + auto result = std::string{ std::begin(bytes), std::end(bytes) }; + BOOST_REQUIRE(expected_result == result); + + result = ""; + for (const auto& ch : bytes) { result.push_back(ch); } + BOOST_REQUIRE(expected_result == result); + + auto extracted_parts = std::vector{}; + offset = size_t{ 0 }; + for (const auto& part : parts) { + extracted_parts.emplace_back(part.size(), '\0'); + std::copy(std::begin(bytes) + offset, std::begin(bytes) + offset + part.size(), + std::begin(extracted_parts.back())); + offset += part.size(); + } + BOOST_REQUIRE(parts == extracted_parts); +} + +BOOST_AUTO_TEST_CASE(next_test) { + static constexpr auto* a_value = "a"; + static constexpr auto* a_next_expected = "b"; + auto a = shared_bytes(a_value, 1); + BOOST_REQUIRE(memcmp(a.next().data(), reinterpret_cast(a_next_expected), 1) == 0); + BOOST_REQUIRE(a.next().size() == 1); + + static constexpr auto* aa_value = "aa"; + static constexpr auto* aa_next_expected = "ab"; + auto aa = shared_bytes(aa_value, 2); + BOOST_REQUIRE(memcmp(aa.next().data(), reinterpret_cast(aa_next_expected), 2) == 0); + BOOST_REQUIRE(aa.next().size() == 2); + + char test_value_1[3] = {static_cast(0x00), static_cast(0x00), static_cast(0xFF)}; + char test_value_1_next_expected[2] = {static_cast(0x00), static_cast(0x01)}; + auto test_value_1_sb = shared_bytes(test_value_1, 3); + BOOST_REQUIRE(test_value_1_sb.next().size() == 2); + BOOST_REQUIRE(memcmp(test_value_1_sb.next().data(), reinterpret_cast(test_value_1_next_expected), 2) == 0); + + char test_value_2[3] = {static_cast(0x00), static_cast(0xFF), static_cast(0xFF)}; + char test_value_2_next_expected[1] = {static_cast(0x01)}; + auto test_value_2_sb = shared_bytes(test_value_2, 3); + BOOST_REQUIRE(test_value_2_sb.next().size() == 1); + BOOST_REQUIRE(memcmp(test_value_2_sb.next().data(), reinterpret_cast(test_value_2_next_expected), 1) == 0); + + static constexpr auto* empty_value = ""; + auto empty = shared_bytes(empty_value, 0); + BOOST_CHECK_THROW(empty.next().size(), eosio::chain::chain_exception); + + // next of a sequence of 0xFFs is empty + char single_last_value[1] = {static_cast(0xFF)}; + auto single_last = shared_bytes(single_last_value, 1); + BOOST_CHECK_THROW(single_last.next().size(), eosio::chain::chain_exception); + + char double_last_value[2] = {static_cast(0xFF), static_cast(0xFF)}; + auto double_last = shared_bytes(double_last_value, 2); + BOOST_CHECK_THROW(double_last.next().size(), eosio::chain::chain_exception); + + char mixed_last_value[2] = {static_cast(0xFE), static_cast(0xFF)}; + char mixed_last_next_expected[1] = {static_cast(0xFF)}; + auto mixed_last = shared_bytes(mixed_last_value, 2); + BOOST_REQUIRE(memcmp(mixed_last.next().data(), reinterpret_cast(mixed_last_next_expected), 1) == 0); + BOOST_REQUIRE(mixed_last.next().size() == 1); +} + +BOOST_AUTO_TEST_CASE(truncate_key_test) { + static constexpr auto* a_value = "a"; + auto a = shared_bytes(a_value, 1); + auto trunc_key_a = shared_bytes::truncate_key(a); + BOOST_REQUIRE(trunc_key_a.size() == 0); + + static constexpr auto* aa_value = "aa"; + static constexpr auto* aa_truncated_expected = "a"; + auto aa = shared_bytes(aa_value, 2); + auto trunc_key_aa = shared_bytes::truncate_key(aa); + BOOST_REQUIRE(memcmp(trunc_key_aa.data(), reinterpret_cast(aa_truncated_expected), 1) == 0); + BOOST_REQUIRE(trunc_key_aa.size() == 1); + + static constexpr auto* empty_value = ""; + auto empty = shared_bytes(empty_value, 0); + BOOST_CHECK_THROW(shared_bytes::truncate_key(empty), eosio::chain::chain_exception); +} + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/libraries/chain_kv/unit_tests/undo_stack_tests.cpp b/libraries/chain_kv/unit_tests/undo_stack_tests.cpp new file mode 100644 index 00000000000..eda20d9e73e --- /dev/null +++ b/libraries/chain_kv/unit_tests/undo_stack_tests.cpp @@ -0,0 +1,456 @@ +#include "chain_kv_tests.hpp" +#include + +using chain_kv::bytes; +using chain_kv::to_slice; + +BOOST_AUTO_TEST_SUITE(undo_stack_tests) + +void undo_tests(bool reload_undo, uint64_t target_segment_size) { + boost::filesystem::remove_all("test-undo-db"); + chain_kv::database db{ "test-undo-db", true }; + std::unique_ptr undo_stack; + + auto reload = [&] { + if (!undo_stack || reload_undo) + undo_stack = std::make_unique(db, bytes{ 0x10 }, target_segment_size); + }; + reload(); + + KV_REQUIRE_EXCEPTION(undo_stack->undo(), "nothing to undo"); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 0); + { + chain_kv::write_session session{ db }; + session.set({ 0x20, 0x00 }, to_slice({})); + session.set({ 0x20, 0x02 }, to_slice({ 0x50 })); + session.set({ 0x20, 0x01 }, to_slice({ 0x40 })); + session.erase({ 0x20, 0x02 }); + session.set({ 0x20, 0x03 }, to_slice({ 0x60 })); + session.set({ 0x20, 0x01 }, to_slice({ 0x50 })); + session.write_changes(*undo_stack); + } + KV_REQUIRE_EXCEPTION(undo_stack->undo(), "nothing to undo"); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 0); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x10, (char)0x80 }), (kv_values{})); // no undo segments + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x00 }, {} }, + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x03 }, { 0x60 } }, + } })); + + reload(); + undo_stack->push(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 1); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 1); + { + chain_kv::write_session session{ db }; + session.erase({ 0x20, 0x01 }); + session.set({ 0x20, 0x00 }, to_slice({ 0x70 })); + session.write_changes(*undo_stack); + } + BOOST_REQUIRE_NE(get_all(db, { 0x10, (char)0x80 }), (kv_values{})); // has undo segments + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x00 }, { 0x70 } }, + { { 0x20, 0x03 }, { 0x60 } }, + } })); + + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 1); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 1); + KV_REQUIRE_EXCEPTION(undo_stack->set_revision(2), "cannot set revision while there is an existing undo stack"); + undo_stack->undo(); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x10, (char)0x80 }), (kv_values{})); // no undo segments + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 0); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 0); + undo_stack->set_revision(10); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 10); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 10); + + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x00 }, {} }, + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x03 }, { 0x60 } }, + } })); + + { + chain_kv::write_session session{ db }; + session.erase({ 0x20, 0x01 }); + session.set({ 0x20, 0x00 }, to_slice({ 0x70 })); + session.write_changes(*undo_stack); + } + BOOST_REQUIRE_EQUAL(get_all(db, { 0x10, (char)0x80 }), (kv_values{})); // no undo segments + reload(); + undo_stack->push(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 11); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 11); + KV_REQUIRE_EXCEPTION(undo_stack->set_revision(12), "cannot set revision while there is an existing undo stack"); + undo_stack->commit(0); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 11); + KV_REQUIRE_EXCEPTION(undo_stack->set_revision(12), "cannot set revision while there is an existing undo stack"); + undo_stack->commit(11); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 11); + reload(); + KV_REQUIRE_EXCEPTION(undo_stack->set_revision(9), "revision cannot decrease"); + undo_stack->set_revision(12); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 12); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 12); + + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x00 }, { 0x70 } }, + { { 0x20, 0x03 }, { 0x60 } }, + } })); +} // undo_tests() + +void squash_tests(bool reload_undo, uint64_t target_segment_size) { + boost::filesystem::remove_all("test-squash-db"); + chain_kv::database db{ "test-squash-db", true }; + std::unique_ptr undo_stack; + + auto reload = [&] { + if (!undo_stack || reload_undo) + undo_stack = std::make_unique(db, bytes{ 0x10 }, target_segment_size); + }; + reload(); + + // set 1 + undo_stack->push(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 1); + { + chain_kv::write_session session{ db }; + session.set({ 0x20, 0x01 }, to_slice({ 0x50 })); + session.set({ 0x20, 0x02 }, to_slice({ 0x60 })); + session.write_changes(*undo_stack); + } + reload(); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x60 } }, + } })); + + // set 2 + undo_stack->push(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 2); + { + chain_kv::write_session session{ db }; + session.erase({ 0x20, 0x01 }); + session.set({ 0x20, 0x02 }, to_slice({ 0x61 })); + session.set({ 0x20, 0x03 }, to_slice({ 0x70 })); + session.set({ 0x20, 0x04 }, to_slice({ 0x10 })); + session.write_changes(*undo_stack); + } + reload(); + undo_stack->push(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 3); + { + chain_kv::write_session session{ db }; + session.set({ 0x20, 0x01 }, to_slice({ 0x50 })); + session.set({ 0x20, 0x02 }, to_slice({ 0x62 })); + session.erase({ 0x20, 0x03 }); + session.set({ 0x20, 0x05 }, to_slice({ 0x05 })); + session.set({ 0x20, 0x06 }, to_slice({ 0x06 })); + session.write_changes(*undo_stack); + } + reload(); + undo_stack->squash(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 2); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x62 } }, + { { 0x20, 0x04 }, { 0x10 } }, + { { 0x20, 0x05 }, { 0x05 } }, + { { 0x20, 0x06 }, { 0x06 } }, + } })); + + // set 3 + undo_stack->push(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 3); + { + chain_kv::write_session session{ db }; + session.set({ 0x20, 0x07 }, to_slice({ 0x07 })); + session.set({ 0x20, 0x08 }, to_slice({ 0x08 })); + session.write_changes(*undo_stack); + } + reload(); + undo_stack->push(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 4); + { + chain_kv::write_session session{ db }; + session.set({ 0x20, 0x09 }, to_slice({ 0x09 })); + session.set({ 0x20, 0x0a }, to_slice({ 0x0a })); + session.write_changes(*undo_stack); + } + reload(); + undo_stack->push(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 5); + { + chain_kv::write_session session{ db }; + session.set({ 0x20, 0x0b }, to_slice({ 0x0b })); + session.set({ 0x20, 0x0c }, to_slice({ 0x0c })); + session.write_changes(*undo_stack); + } + reload(); + undo_stack->squash(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 4); + undo_stack->squash(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 3); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x62 } }, + { { 0x20, 0x04 }, { 0x10 } }, + { { 0x20, 0x05 }, { 0x05 } }, + { { 0x20, 0x06 }, { 0x06 } }, + { { 0x20, 0x07 }, { 0x07 } }, + { { 0x20, 0x08 }, { 0x08 } }, + { { 0x20, 0x09 }, { 0x09 } }, + { { 0x20, 0x0a }, { 0x0a } }, + { { 0x20, 0x0b }, { 0x0b } }, + { { 0x20, 0x0c }, { 0x0c } }, + } })); + + // undo set 3 + undo_stack->undo(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 2); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x62 } }, + { { 0x20, 0x04 }, { 0x10 } }, + { { 0x20, 0x05 }, { 0x05 } }, + { { 0x20, 0x06 }, { 0x06 } }, + } })); + + // undo set 2 + undo_stack->undo(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 1); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x60 } }, + } })); + + // undo set 1 + undo_stack->undo(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 0); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ {} })); + + // TODO: test squash with only 1 undo level +} // squash_tests() + +void commit_tests(bool reload_undo, uint64_t target_segment_size) { + boost::filesystem::remove_all("test-commit-db"); + chain_kv::database db{ "test-commit-db", true }; + std::unique_ptr undo_stack; + + auto reload = [&] { + if (!undo_stack || reload_undo) + undo_stack = std::make_unique(db, bytes{ 0x10 }, target_segment_size); + }; + reload(); + + // revision 1 + undo_stack->push(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 1); + { + chain_kv::write_session session{ db }; + session.set({ 0x20, 0x01 }, to_slice({ 0x50 })); + session.set({ 0x20, 0x02 }, to_slice({ 0x60 })); + session.write_changes(*undo_stack); + } + reload(); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x60 } }, + } })); + + // revision 2 + undo_stack->push(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 2); + { + chain_kv::write_session session{ db }; + session.erase({ 0x20, 0x01 }); + session.set({ 0x20, 0x02 }, to_slice({ 0x61 })); + session.set({ 0x20, 0x03 }, to_slice({ 0x70 })); + session.set({ 0x20, 0x04 }, to_slice({ 0x10 })); + session.set({ 0x20, 0x01 }, to_slice({ 0x50 })); + session.set({ 0x20, 0x02 }, to_slice({ 0x62 })); + session.erase({ 0x20, 0x03 }); + session.set({ 0x20, 0x05 }, to_slice({ 0x05 })); + session.set({ 0x20, 0x06 }, to_slice({ 0x06 })); + session.write_changes(*undo_stack); + } + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 2); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x62 } }, + { { 0x20, 0x04 }, { 0x10 } }, + { { 0x20, 0x05 }, { 0x05 } }, + { { 0x20, 0x06 }, { 0x06 } }, + } })); + + // revision 3 + undo_stack->push(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 3); + { + chain_kv::write_session session{ db }; + session.set({ 0x20, 0x07 }, to_slice({ 0x07 })); + session.set({ 0x20, 0x08 }, to_slice({ 0x08 })); + session.set({ 0x20, 0x09 }, to_slice({ 0x09 })); + session.set({ 0x20, 0x0a }, to_slice({ 0x0a })); + session.set({ 0x20, 0x0b }, to_slice({ 0x0b })); + session.set({ 0x20, 0x0c }, to_slice({ 0x0c })); + session.write_changes(*undo_stack); + } + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 3); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x62 } }, + { { 0x20, 0x04 }, { 0x10 } }, + { { 0x20, 0x05 }, { 0x05 } }, + { { 0x20, 0x06 }, { 0x06 } }, + { { 0x20, 0x07 }, { 0x07 } }, + { { 0x20, 0x08 }, { 0x08 } }, + { { 0x20, 0x09 }, { 0x09 } }, + { { 0x20, 0x0a }, { 0x0a } }, + { { 0x20, 0x0b }, { 0x0b } }, + { { 0x20, 0x0c }, { 0x0c } }, + } })); + + // commit revision 1 + undo_stack->commit(1); + + // undo revision 3 + undo_stack->undo(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 2); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x62 } }, + { { 0x20, 0x04 }, { 0x10 } }, + { { 0x20, 0x05 }, { 0x05 } }, + { { 0x20, 0x06 }, { 0x06 } }, + } })); + + // undo revision 2 + undo_stack->undo(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 1); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x60 } }, + } })); + + // Can't undo revision 1 + KV_REQUIRE_EXCEPTION(undo_stack->undo(), "nothing to undo"); + + BOOST_REQUIRE_EQUAL(get_all(db, { 0x10, (char)0x80 }), (kv_values{})); // no undo segments + + // revision 2 + undo_stack->push(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 2); + { + chain_kv::write_session session{ db }; + session.erase({ 0x20, 0x01 }); + session.set({ 0x20, 0x02 }, to_slice({ 0x61 })); + session.set({ 0x20, 0x03 }, to_slice({ 0x70 })); + session.set({ 0x20, 0x04 }, to_slice({ 0x10 })); + session.set({ 0x20, 0x01 }, to_slice({ 0x50 })); + session.set({ 0x20, 0x02 }, to_slice({ 0x62 })); + session.erase({ 0x20, 0x03 }); + session.set({ 0x20, 0x05 }, to_slice({ 0x05 })); + session.set({ 0x20, 0x06 }, to_slice({ 0x06 })); + session.write_changes(*undo_stack); + } + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 2); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x62 } }, + { { 0x20, 0x04 }, { 0x10 } }, + { { 0x20, 0x05 }, { 0x05 } }, + { { 0x20, 0x06 }, { 0x06 } }, + } })); + + // revision 3 + undo_stack->push(); + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 3); + { + chain_kv::write_session session{ db }; + session.set({ 0x20, 0x07 }, to_slice({ 0x07 })); + session.set({ 0x20, 0x08 }, to_slice({ 0x08 })); + session.set({ 0x20, 0x09 }, to_slice({ 0x09 })); + session.set({ 0x20, 0x0a }, to_slice({ 0x0a })); + session.set({ 0x20, 0x0b }, to_slice({ 0x0b })); + session.set({ 0x20, 0x0c }, to_slice({ 0x0c })); + session.write_changes(*undo_stack); + } + reload(); + + // commit revision 3 + undo_stack->commit(3); + + // Can't undo + KV_REQUIRE_EXCEPTION(undo_stack->undo(), "nothing to undo"); + + BOOST_REQUIRE_EQUAL(get_all(db, { 0x10, (char)0x80 }), (kv_values{})); // no undo segments + + reload(); + BOOST_REQUIRE_EQUAL(undo_stack->revision(), 3); + BOOST_REQUIRE_EQUAL(get_all(db, { 0x20 }), (kv_values{ { + { { 0x20, 0x01 }, { 0x50 } }, + { { 0x20, 0x02 }, { 0x62 } }, + { { 0x20, 0x04 }, { 0x10 } }, + { { 0x20, 0x05 }, { 0x05 } }, + { { 0x20, 0x06 }, { 0x06 } }, + { { 0x20, 0x07 }, { 0x07 } }, + { { 0x20, 0x08 }, { 0x08 } }, + { { 0x20, 0x09 }, { 0x09 } }, + { { 0x20, 0x0a }, { 0x0a } }, + { { 0x20, 0x0b }, { 0x0b } }, + { { 0x20, 0x0c }, { 0x0c } }, + } })); +} // commit_tests() + +BOOST_AUTO_TEST_CASE(test_undo) { + undo_tests(false, 0); + undo_tests(true, 0); + undo_tests(false, 64 * 1024 * 1024); + undo_tests(true, 64 * 1024 * 1024); +} + +BOOST_AUTO_TEST_CASE(test_squash) { + squash_tests(false, 0); + squash_tests(true, 0); + squash_tests(false, 64 * 1024 * 1024); + squash_tests(true, 64 * 1024 * 1024); +} + +BOOST_AUTO_TEST_CASE(test_commit) { + commit_tests(false, 0); + commit_tests(true, 0); + commit_tests(false, 64 * 1024 * 1024); + commit_tests(true, 64 * 1024 * 1024); +} + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/libraries/chain_kv/unit_tests/view_tests.cpp b/libraries/chain_kv/unit_tests/view_tests.cpp new file mode 100644 index 00000000000..2296b3ff213 --- /dev/null +++ b/libraries/chain_kv/unit_tests/view_tests.cpp @@ -0,0 +1,216 @@ +#include "chain_kv_tests.hpp" +#include + +using chain_kv::bytes; +using chain_kv::to_slice; + +BOOST_AUTO_TEST_SUITE(view_tests) + +void view_test(bool reload_session) { + boost::filesystem::remove_all("view-test-session-db"); + chain_kv::database db{ "view-test-session-db", true }; + chain_kv::undo_stack undo_stack{ db, { 0x10 } }; + std::unique_ptr session; + std::unique_ptr view; + + auto reload = [&] { + if (session && reload_session) { + session->write_changes(undo_stack); + view = nullptr; + session = nullptr; + } + if (!session) + session = std::make_unique(db); + if (!view) + view = std::make_unique(*session, bytes{ 0x70 }); + }; + reload(); + + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x1234), (kv_values{})); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x1234), get_matching2(*view, 0x1234)); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x5678), (kv_values{})); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x5678), get_matching2(*view, 0x5678)); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x9abc), (kv_values{})); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x9abc), get_matching2(*view, 0x9abc)); + + view->set(0x1234, to_slice({ 0x30, 0x40 }), to_slice({ 0x50, 0x60 })); + view->set(0x5678, to_slice({ 0x30, 0x71 }), to_slice({ 0x59, 0x69 })); + view->set(0x5678, to_slice({ 0x30, 0x00 }), to_slice({ 0x59, 0x69 })); + view->set(0x5678, to_slice({ 0x30, 0x42 }), to_slice({ 0x55, 0x66 })); + view->set(0x5678, to_slice({ 0x30, 0x41 }), to_slice({ 0x51, 0x61 })); + view->set(0x9abc, to_slice({ 0x30, 0x42 }), to_slice({ 0x52, 0x62 })); + reload(); + + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x1234), (kv_values{ { + { { 0x30, 0x40 }, { 0x50, 0x60 } }, + } })); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x1234), get_matching2(*view, 0x1234)); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x5678), (kv_values{ { + { { 0x30, 0x00 }, { 0x59, 0x69 } }, + { { 0x30, 0x41 }, { 0x51, 0x61 } }, + { { 0x30, 0x42 }, { 0x55, 0x66 } }, + { { 0x30, 0x71 }, { 0x59, 0x69 } }, + } })); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x5678), get_matching2(*view, 0x5678)); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x9abc), (kv_values{ { + { { 0x30, 0x42 }, { 0x52, 0x62 } }, + } })); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x9abc), get_matching2(*view, 0x9abc)); + + view->erase(0x5678, to_slice({ 0x30, 0x00 })); + view->erase(0x5678, to_slice({ 0x30, 0x71 })); + view->erase(0x5678, to_slice({ 0x30, 0x42 })); + reload(); + + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x1234), (kv_values{ { + { { 0x30, 0x40 }, { 0x50, 0x60 } }, + } })); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x1234), get_matching2(*view, 0x1234)); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x5678), (kv_values{ { + { { 0x30, 0x41 }, { 0x51, 0x61 } }, + } })); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x5678), get_matching2(*view, 0x5678)); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x9abc), (kv_values{ { + { { 0x30, 0x42 }, { 0x52, 0x62 } }, + } })); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0x9abc), get_matching2(*view, 0x9abc)); + + { + chain_kv::view::iterator it{ *view, 0x5678, {} }; + it.lower_bound({ 0x30, 0x22 }); + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { 0x30, 0x41 }, { 0x51, 0x61 } }, + } })); + view->set(0x5678, to_slice({ 0x30, 0x22 }), to_slice({ 0x55, 0x66 })); + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { 0x30, 0x41 }, { 0x51, 0x61 } }, + } })); + it.lower_bound({ 0x30, 0x22 }); + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { 0x30, 0x22 }, { 0x55, 0x66 } }, + } })); + view->set(0x5678, to_slice({ 0x30, 0x22 }), to_slice({ 0x00, 0x11 })); + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { 0x30, 0x22 }, { 0x00, 0x11 } }, + } })); + view->erase(0x5678, to_slice({ 0x30, 0x22 })); + KV_REQUIRE_EXCEPTION(it.get_kv(), "kv iterator is at an erased value"); + KV_REQUIRE_EXCEPTION(--it, "kv iterator is at an erased value"); + KV_REQUIRE_EXCEPTION(++it, "kv iterator is at an erased value"); + view->set(0x5678, to_slice({ 0x30, 0x22 }), to_slice({})); + KV_REQUIRE_EXCEPTION(it.get_kv(), "kv iterator is at an erased value"); + it.lower_bound({ 0x30, 0x22 }); + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { 0x30, 0x22 }, {} }, + } })); + view->set(0x5678, to_slice({ 0x30, 0x22 }), to_slice({ 0x00 })); + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { 0x30, 0x22 }, { 0x00 } }, + } })); + view->erase(0x5678, to_slice({ 0x30, 0x22 })); + KV_REQUIRE_EXCEPTION(it.get_kv(), "kv iterator is at an erased value"); + } + reload(); + + { + chain_kv::view::iterator it{ *view, 0xbeefbeef, {} }; + view->set(0xbeefbeef, to_slice({ (char)0x80 }), to_slice({ (char)0xff })); + view->set(0xbeefbeef, to_slice({ (char)0x90 }), to_slice({ (char)0xfe })); + view->set(0xbeefbeef, to_slice({ (char)0xa0 }), to_slice({ (char)0xfd })); + view->set(0xbeefbeef, to_slice({ (char)0xb0 }), to_slice({ (char)0xfc })); + it.lower_bound({}); + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { (char)0x80 }, { (char)0xff } }, + } })); + it.lower_bound({ (char)0x80 }); + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { (char)0x80 }, { (char)0xff } }, + } })); + it.lower_bound({ (char)0x81 }); + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { (char)0x90 }, { (char)0xfe } }, + } })); + it.lower_bound({ (char)0x90 }); + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { (char)0x90 }, { (char)0xfe } }, + } })); + --it; + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { (char)0x80 }, { (char)0xff } }, + } })); + ++it; + ++it; + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { (char)0xa0 }, { (char)0xfd } }, + } })); + view->erase(0xbeefbeef, to_slice({ (char)0x90 })); + --it; + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { (char)0x80 }, { (char)0xff } }, + } })); + view->erase(0xbeefbeef, to_slice({ (char)0xa0 })); + ++it; + BOOST_REQUIRE_EQUAL(get_it(it), (kv_values{ { + { { (char)0xb0 }, { (char)0xfc } }, + } })); + } + reload(); + + BOOST_REQUIRE_EQUAL(get_matching(*view, 0xbeefbeef), (kv_values{ { + { { (char)0x80 }, { (char)0xff } }, + { { (char)0xb0 }, { (char)0xfc } }, + } })); + + view->set(0xf00df00d, to_slice({ 0x10, 0x20, 0x00 }), to_slice({ (char)0x70 })); + view->set(0xf00df00d, to_slice({ 0x10, 0x20, 0x01 }), to_slice({ (char)0x71 })); + view->set(0xf00df00d, to_slice({ 0x10, 0x20, 0x02 }), to_slice({ (char)0x72 })); + + view->set(0xf00df00d, to_slice({ 0x10, 0x30, 0x00 }), to_slice({ (char)0x70 })); + view->set(0xf00df00d, to_slice({ 0x10, 0x30, 0x01 }), to_slice({ (char)0x71 })); + view->set(0xf00df00d, to_slice({ 0x10, 0x30, 0x02 }), to_slice({ (char)0x72 })); + + view->set(0xf00df00d, to_slice({ 0x20, 0x00 }), to_slice({ (char)0x70 })); + view->set(0xf00df00d, to_slice({ 0x20, 0x01 }), to_slice({ (char)0x71 })); + view->set(0xf00df00d, to_slice({ 0x20, 0x02 }), to_slice({ (char)0x72 })); + reload(); + + BOOST_REQUIRE_EQUAL(get_matching(*view, 0xf00df00d, { 0x10, 0x20 }), (kv_values{ { + { { 0x10, 0x20, 0x00 }, { 0x70 } }, + { { 0x10, 0x20, 0x01 }, { 0x71 } }, + { { 0x10, 0x20, 0x02 }, { 0x72 } }, + } })); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0xf00df00d, { 0x10, 0x20 }), + get_matching2(*view, 0xf00df00d, { 0x10, 0x20 })); + + BOOST_REQUIRE_EQUAL(get_matching(*view, 0xf00df00d, { 0x10, 0x30 }), (kv_values{ { + { { 0x10, 0x30, 0x00 }, { 0x70 } }, + { { 0x10, 0x30, 0x01 }, { 0x71 } }, + { { 0x10, 0x30, 0x02 }, { 0x72 } }, + } })); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0xf00df00d, { 0x10, 0x30 }), + get_matching2(*view, 0xf00df00d, { 0x10, 0x30 })); + + BOOST_REQUIRE_EQUAL(get_matching(*view, 0xf00df00d, { 0x10 }), (kv_values{ { + { { 0x10, 0x20, 0x00 }, { 0x70 } }, + { { 0x10, 0x20, 0x01 }, { 0x71 } }, + { { 0x10, 0x20, 0x02 }, { 0x72 } }, + { { 0x10, 0x30, 0x00 }, { 0x70 } }, + { { 0x10, 0x30, 0x01 }, { 0x71 } }, + { { 0x10, 0x30, 0x02 }, { 0x72 } }, + } })); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0xf00df00d, { 0x10 }), get_matching2(*view, 0xf00df00d, { 0x10 })); + + BOOST_REQUIRE_EQUAL(get_matching(*view, 0xf00df00d, { 0x20 }), (kv_values{ { + { { 0x20, 0x00 }, { 0x70 } }, + { { 0x20, 0x01 }, { 0x71 } }, + { { 0x20, 0x02 }, { 0x72 } }, + } })); + BOOST_REQUIRE_EQUAL(get_matching(*view, 0xf00df00d, { 0x20 }), get_matching2(*view, 0xf00df00d, { 0x20 })); +} // view_test() + +BOOST_AUTO_TEST_CASE(test_view) { + view_test(false); + view_test(true); +} + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/libraries/chain_kv/unit_tests/write_session_tests.cpp b/libraries/chain_kv/unit_tests/write_session_tests.cpp new file mode 100644 index 00000000000..5201f2cca80 --- /dev/null +++ b/libraries/chain_kv/unit_tests/write_session_tests.cpp @@ -0,0 +1,71 @@ +#include "chain_kv_tests.hpp" +#include + +using chain_kv::bytes; +using chain_kv::to_slice; + +BOOST_AUTO_TEST_SUITE(write_session_tests) + +void write_session_test(bool reload_session) { + boost::filesystem::remove_all("test-write-session-db"); + chain_kv::database db{ "test-write-session-db", true }; + chain_kv::undo_stack undo_stack{ db, { 0x10 } }; + std::unique_ptr session; + + auto reload = [&] { + if (session && reload_session) { + session->write_changes(undo_stack); + session = nullptr; + } + if (!session) + session = std::make_unique(db); + }; + reload(); + + std::vector keys = { + { 0x20 }, + { 0x20, 0x00 }, + { 0x30 }, + }; + + BOOST_REQUIRE_EQUAL(get_values(*session, keys), (kv_values{})); + session->erase({ 0x20 }); + reload(); + BOOST_REQUIRE_EQUAL(get_values(*session, keys), (kv_values{})); + session->set({ 0x20, 0x00 }, to_slice({ 0x01 })); + session->set({ 0x30 }, to_slice({ 0x02 })); + reload(); + session->set({ 0x20 }, to_slice({ 0x03 })); + reload(); + BOOST_REQUIRE_EQUAL(get_values(*session, keys), (kv_values{ { + { { 0x20 }, { 0x03 } }, + { { 0x20, 0x00 }, { 0x01 } }, + { { 0x30 }, { 0x02 } }, + } })); + session->erase({ 0x20 }); + reload(); + BOOST_REQUIRE_EQUAL(get_values(*session, keys), (kv_values{ { + { { 0x20, 0x00 }, { 0x01 } }, + { { 0x30 }, { 0x02 } }, + } })); + session->set({ 0x20, 0x00 }, to_slice({ 0x11 })); + reload(); + BOOST_REQUIRE_EQUAL(get_values(*session, keys), (kv_values{ { + { { 0x20, 0x00 }, { 0x11 } }, + { { 0x30 }, { 0x02 } }, + } })); + session->set({ 0x20 }, to_slice({ 0x23 })); + reload(); + BOOST_REQUIRE_EQUAL(get_values(*session, keys), (kv_values{ { + { { 0x20 }, { 0x23 } }, + { { 0x20, 0x00 }, { 0x11 } }, + { { 0x30 }, { 0x02 } }, + } })); +} // write_session_test() + +BOOST_AUTO_TEST_CASE(test_write_session) { + write_session_test(false); + write_session_test(true); +} + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/libraries/chainbase b/libraries/chainbase index eccb1aaa2fc..8a131e30fa6 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit eccb1aaa2fc51d91d836e7bf4198b006452e7e16 +Subproject commit 8a131e30fa60ecfef977a380d301cccbfe5929ab diff --git a/libraries/eos-vm b/libraries/eos-vm index 0b8c291c082..d42a1815266 160000 --- a/libraries/eos-vm +++ b/libraries/eos-vm @@ -1 +1 @@ -Subproject commit 0b8c291c08280394cab1a21f88a8cb24ebd1050b +Subproject commit d42a18152669c1cec69ad43e82af41c276bb4172 diff --git a/libraries/fc b/libraries/fc index a3694752d2d..863dc8d371f 160000 --- a/libraries/fc +++ b/libraries/fc @@ -1 +1 @@ -Subproject commit a3694752d2dd2021185955eafc1a504efaabfa79 +Subproject commit 863dc8d371fd4da25f89cb08b13737f009a9cec7 diff --git a/libraries/rocksdb b/libraries/rocksdb new file mode 160000 index 00000000000..551a1109184 --- /dev/null +++ b/libraries/rocksdb @@ -0,0 +1 @@ +Subproject commit 551a110918493a19d11243f53408b97485de1411 diff --git a/libraries/rodeos/.clang-format b/libraries/rodeos/.clang-format new file mode 100644 index 00000000000..fa257fc706b --- /dev/null +++ b/libraries/rodeos/.clang-format @@ -0,0 +1,76 @@ +BasedOnStyle: LLVM +IndentWidth: 3 +UseTab: Never +ColumnLimit: 120 + +--- +Language: Cpp +# always align * and & to the type +DerivePointerAlignment: false +PointerAlignment: Left + +# regroup includes to these classes +IncludeCategories: + - Regex: '(<|"(eosio)/)' + Priority: 4 + - Regex: '(<|"(boost)/)' + Priority: 3 + - Regex: '(<|"(llvm|llvm-c|clang|clang-c)/' + Priority: 3 + - Regex: '<[[:alnum:]]+>' + Priority: 2 + - Regex: '.*' + Priority: 1 + +#IncludeBlocks: Regroup + +# set indent for public, private and protected +#AccessModifierOffset: 3 + +# make line continuations twice the normal indent +ContinuationIndentWidth: 6 + +# add missing namespace comments +FixNamespaceComments: true + +# add spaces to braced list i.e. int* foo = { 0, 1, 2 }; instead of int* foo = {0,1,2}; +Cpp11BracedListStyle: false +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: true +AlignConsecutiveDeclarations: true +AlignOperands: true +AlignTrailingComments: true +AllowShortCaseLabelsOnASingleLine: true +AllowShortFunctionsOnASingleLine: All +AllowShortBlocksOnASingleLine: true +#AllowShortIfStatementsOnASingleLine: WithoutElse +#AllowShortIfStatementsOnASingleLine: true +#AllowShortLambdasOnASingleLine: All +AllowShortLoopsOnASingleLine: true +AlwaysBreakTemplateDeclarations: true + +BinPackParameters: true +### use this with clang9 +BreakBeforeBraces: Custom +BraceWrapping: + #AfterCaseLabel: true + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + +BreakConstructorInitializers: BeforeColon +CompactNamespaces: true +IndentCaseLabels: true +IndentPPDirectives: AfterHash +NamespaceIndentation: None +ReflowComments: false +SortIncludes: true +SortUsingDeclarations: true +--- diff --git a/libraries/rodeos/CMakeLists.txt b/libraries/rodeos/CMakeLists.txt new file mode 100644 index 00000000000..b8004b98e82 --- /dev/null +++ b/libraries/rodeos/CMakeLists.txt @@ -0,0 +1,16 @@ +file(GLOB_RECURSE HEADERS "include/*.hpp" "include/*.h") + +add_library( rodeos_lib + embedded_rodeos.cpp + rodeos.cpp + wasm_ql.cpp + ${HEADERS} + ) + +target_link_libraries( rodeos_lib + PUBLIC abieos chain_kv eosio_chain fc softfloat + ) + +target_include_directories( rodeos_lib + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" + ) \ No newline at end of file diff --git a/libraries/rodeos/embedded_rodeos.cpp b/libraries/rodeos/embedded_rodeos.cpp new file mode 100644 index 00000000000..f5f560624d1 --- /dev/null +++ b/libraries/rodeos/embedded_rodeos.cpp @@ -0,0 +1,289 @@ +#include +#include +#include + +struct rodeos_error_s { + const char* msg = "no error"; + std::string buffer; + + bool set(const char* m) { + try { + buffer = m; + msg = buffer.c_str(); + } catch (...) { msg = "error storing error message"; } + return false; + } +}; + +struct rodeos_context_s : b1::rodeos::rodeos_context {}; + +struct rodeos_db_partition_s { + std::shared_ptr obj; +}; + +struct rodeos_db_snapshot_s : b1::rodeos::rodeos_db_snapshot { + using rodeos_db_snapshot::rodeos_db_snapshot; +}; + +struct rodeos_filter_s : b1::rodeos::rodeos_filter { + using rodeos_filter::rodeos_filter; +}; + +struct rodeos_query_handler_s : b1::rodeos::rodeos_query_handler { + using rodeos_query_handler::rodeos_query_handler; +}; + +extern "C" rodeos_error* rodeos_create_error() { + try { + return std::make_unique().release(); + } catch (...) { return nullptr; } +} + +extern "C" void rodeos_destroy_error(rodeos_error* error) { std::unique_ptr{ error }; } + +extern "C" const char* rodeos_get_error(rodeos_error* error) { + if (!error) + return "error is null"; + return error->msg; +} + +template +auto handle_exceptions(rodeos_error* error, T errval, F f) noexcept -> decltype(f()) { + if (!error) + return errval; + try { + return f(); + } catch (std::exception& e) { + error->set(e.what()); + return errval; + } catch (...) { + error->set("unknown exception"); + return errval; + } +} + +extern "C" rodeos_context* rodeos_create() { + try { + return std::make_unique().release(); + } catch (...) { return nullptr; } +} + +extern "C" void rodeos_destroy(rodeos_context* context) { std::unique_ptr{ context }; } + +extern "C" rodeos_bool rodeos_open_db(rodeos_error* error, rodeos_context* context, const char* path, + rodeos_bool create_if_missing, int num_threads, int max_open_files) { + return handle_exceptions(error, false, [&] { + if (!context) + return error->set("context is null"); + if (!path) + return error->set("path is null"); + if (context->db) + return error->set("a database is already open on this context"); + context->db = std::make_shared( + path, create_if_missing, num_threads ? std::make_optional(num_threads) : std::nullopt, + max_open_files ? std::make_optional(max_open_files) : std::nullopt); + return true; + }); +} + +extern "C" rodeos_db_partition* rodeos_create_partition(rodeos_error* error, rodeos_context* context, + const char* prefix, uint32_t prefix_size) { + return handle_exceptions(error, nullptr, [&]() -> rodeos_db_partition* { + if (!context) + return error->set("context is null"), nullptr; + if (!prefix) + return error->set("prefix is null"), nullptr; + if (!context->db) + return error->set("database wasn't opened"), nullptr; + auto p = std::make_unique(); + p->obj = std::make_shared(context->db, + std::vector{ prefix, prefix + prefix_size }); + return p.release(); + }); +} + +extern "C" void rodeos_destroy_partition(rodeos_db_partition* partition) { + std::unique_ptr{ partition }; +} + +extern "C" rodeos_db_snapshot* rodeos_create_snapshot(rodeos_error* error, rodeos_db_partition* partition, + rodeos_bool persistent) { + return handle_exceptions(error, nullptr, [&]() -> rodeos_db_snapshot* { + if (!partition) + return error->set("partition is null"), nullptr; + return std::make_unique(partition->obj, persistent).release(); + }); +} + +extern "C" void rodeos_destroy_snapshot(rodeos_db_snapshot* snapshot) { + std::unique_ptr{ snapshot }; +} + +extern "C" rodeos_bool rodeos_refresh_snapshot(rodeos_error* error, rodeos_db_snapshot* snapshot) { + return handle_exceptions(error, false, [&]() { + if (!snapshot) + return error->set("snapshot is null"); + snapshot->refresh(); + return true; + }); +} + +template +void with_result(const char* data, uint64_t size, F f) { + eosio::input_stream bin{ data, data + size }; + eosio::ship_protocol::result result; + from_bin(result, bin); + auto* result_v0 = std::get_if(&result); + if (result_v0) + return f(*result_v0); + + auto* result_v1 = std::get_if(&result); + if (result_v1) + return f(*result_v1); + + throw std::runtime_error("expected a get_blocks_result_v0 or get_blocks_result_v1"); +} + +extern "C" rodeos_bool rodeos_start_block(rodeos_error* error, rodeos_db_snapshot* snapshot, const char* data, + uint64_t size) { + return handle_exceptions(error, false, [&]() { + if (!snapshot) + return error->set("snapshot is null"); + with_result(data, size, [&](auto& result) { snapshot->start_block(result); }); + return true; + }); +} + +extern "C" rodeos_bool rodeos_end_block(rodeos_error* error, rodeos_db_snapshot* snapshot, const char* data, + uint64_t size, bool force_write) { + return handle_exceptions(error, false, [&]() { + if (!snapshot) + return error->set("snapshot is null"); + with_result(data, size, [&](auto& result) { snapshot->end_block(result, force_write); }); + return true; + }); +} + +extern "C" rodeos_bool rodeos_write_block_info(rodeos_error* error, rodeos_db_snapshot* snapshot, const char* data, + uint64_t size) { + return handle_exceptions(error, false, [&]() { + if (!snapshot) + return error->set("snapshot is null"); + with_result(data, size, [&](auto& result) { snapshot->write_block_info(result); }); + return true; + }); +} + +extern "C" rodeos_bool rodeos_write_deltas(rodeos_error* error, rodeos_db_snapshot* snapshot, const char* data, + uint64_t size, rodeos_bool (*shutdown)(void*), void* shutdown_arg) { + return handle_exceptions(error, false, [&]() { + if (!snapshot) + return error->set("snapshot is null"); + with_result(data, size, [&](auto& result) { + snapshot->write_deltas(result, [=]() -> bool { + if (shutdown) + return shutdown(shutdown_arg); + else + return false; + }); + }); + return true; + }); +} + +extern "C" rodeos_filter* rodeos_create_filter(rodeos_error* error, uint64_t name, const char* wasm_filename) { + return handle_exceptions(error, nullptr, [&]() -> rodeos_filter* { // + return std::make_unique(eosio::name{ name }, wasm_filename).release(); + }); +} + +extern "C" void rodeos_destroy_filter(rodeos_filter* filter) { std::unique_ptr{ filter }; } + +extern "C" rodeos_bool rodeos_run_filter(rodeos_error* error, rodeos_db_snapshot* snapshot, rodeos_filter* filter, + const char* data, uint64_t size, + rodeos_bool (*push_data)(void* arg, const char* data, uint64_t size), + void* push_data_arg) { + return handle_exceptions(error, false, [&]() { + if (!snapshot) + return error->set("snapshot is null"); + if (!filter) + return error->set("filter is null"); + with_result(data, size, [&](auto& result) { + filter->process(*snapshot, result, { data, data + size }, [&](const char* data, uint64_t size) { + if (push_data && !push_data(push_data_arg, data, size)) + throw std::runtime_error("push_data returned false"); + }); + }); + return true; + }); +} + +extern "C" rodeos_query_handler* rodeos_create_query_handler(rodeos_error* error, rodeos_db_partition* partition, + uint32_t max_console_size, uint32_t wasm_cache_size, + uint64_t max_exec_time_ms, const char* contract_dir) { + return handle_exceptions(error, nullptr, [&]() -> rodeos_query_handler* { + if (!partition) + return error->set("partition is null"), nullptr; + auto shared_state = std::make_shared(partition->obj->db); + shared_state->max_console_size = max_console_size; + shared_state->wasm_cache_size = wasm_cache_size; + shared_state->max_exec_time_ms = max_exec_time_ms; + shared_state->contract_dir = contract_dir ? contract_dir : ""; + return std::make_unique(partition->obj, shared_state).release(); + }); +} + +void rodeos_destroy_query_handler(rodeos_query_handler* handler) { std::unique_ptr{ handler }; } + +rodeos_bool rodeos_query_transaction(rodeos_error* error, rodeos_query_handler* handler, rodeos_db_snapshot* snapshot, + const char* data, uint64_t size, char** result, uint64_t* result_size) { + return handle_exceptions(error, false, [&]() { + if (!handler) + return error->set("handler is null"); + if (!result) + return error->set("result is null"); + if (!result_size) + return error->set("result_size is null"); + *result = nullptr; + *result_size = 0; + + std::vector> memory; + eosio::input_stream s{ data, size }; + auto trx = eosio::from_bin(s); + + auto thread_state = handler->state_cache.get_state(); + eosio::ship_protocol::transaction_trace tt; + if (snapshot->snap) { + tt = query_send_transaction(*thread_state, snapshot->partition->contract_kv_prefix, trx, + snapshot->snap->snapshot(), memory, true); + } else { + tt = query_send_transaction(*thread_state, snapshot->partition->contract_kv_prefix, trx, nullptr, memory, + true); + } + + handler->state_cache.store_state(std::move(thread_state)); + + eosio::size_stream ss; + eosio::to_bin(tt, ss); + *result = (char*)malloc(ss.size); + if (!result) + throw std::bad_alloc(); + auto free_on_except = fc::make_scoped_exit([&]{ + free(*result); + *result = nullptr; + }); + eosio::fixed_buf_stream fbs(*result, ss.size); + to_bin(tt, fbs); + if (fbs.pos != fbs.end) { + eosio::check(false, eosio::convert_stream_error(eosio::stream_error::underrun)); + } + *result_size = ss.size; + free_on_except.cancel(); + return true; + }); +} + +void rodeos_free_result(char* result) { + if (result) + free(result); +} diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/action.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/action.hpp new file mode 100644 index 00000000000..8cf77e7cf20 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/action.hpp @@ -0,0 +1,54 @@ +#pragma once + +#include +#include +#include +#include + +namespace b1::rodeos { + +struct action_state { + eosio::name receiver{}; + eosio::input_stream action_data{}; + std::vector action_return_value{}; +}; + +template +struct action_callbacks { + Derived& derived() { return static_cast(*this); } + + int read_action_data(eosio::vm::span data) { + auto& state = derived().get_state(); + size_t s = state.action_data.end - state.action_data.pos; + memcpy(data.data(), state.action_data.pos, std::min(data.size(), s)); + return s; + } + + int action_data_size() { + auto& state = derived().get_state(); + return state.action_data.end - state.action_data.pos; + } + + uint64_t current_receiver() { return derived().get_state().receiver.value; } + + void set_action_return_value(eosio::vm::span packed_blob) { + uint32_t max_action_return_value_size = + derived().get_state().shared->max_action_return_value_size; + EOS_ASSERT(packed_blob.size() <= max_action_return_value_size, + eosio::chain::action_return_value_exception, + "action return value size must be less than ${s} bytes", + ("s", max_action_return_value_size)); + derived().get_state().action_return_value.assign(packed_blob.begin(), packed_blob.end()); + } + + template + static void register_callbacks() { + // todo: preconditions + Rft::template add<&Derived::read_action_data>("env", "read_action_data"); + Rft::template add<&Derived::action_data_size>("env", "action_data_size"); + Rft::template add<&Derived::current_receiver>("env", "current_receiver"); + Rft::template add<&Derived::set_action_return_value>("env", "set_action_return_value"); + } +}; // action_callbacks + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/basic.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/basic.hpp new file mode 100644 index 00000000000..12c0b499175 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/basic.hpp @@ -0,0 +1,71 @@ +#pragma once + +#include +#include + +namespace b1::rodeos { + +struct assert_exception : std::exception { + std::string msg; + + assert_exception(std::string&& msg) : msg(std::move(msg)) {} + + const char* what() const noexcept override { return msg.c_str(); } +}; + +template +struct context_free_system_callbacks { + Derived& derived() { return static_cast(*this); } + + void abort() { throw std::runtime_error("called abort"); } + + void eosio_assert(bool condition, null_terminated_ptr msg) { + if (!condition) + throw assert_exception(std::string(msg.data(), msg.size())); + } + + void eosio_assert_message(bool condition, legacy_span msg) { + if (!condition) + throw assert_exception(std::string(msg.data(), msg.size())); + } + + template + static void register_callbacks() { + // todo: preconditions + Rft::template add<&Derived::abort>("env", "abort"); + Rft::template add<&Derived::eosio_assert>("env", "eosio_assert"); + Rft::template add<&Derived::eosio_assert_message>("env", "eosio_assert_message"); + } +}; + +template +struct data_state { + eosio::input_stream input_data; + std::vector output_data; +}; + +template +struct data_callbacks { + Derived& derived() { return static_cast(*this); } + + uint32_t get_input_data(eosio::vm::span dest) { + auto& input_data = derived().get_state().input_data; + memcpy(dest.data(), input_data.pos, std::min(dest.size(), size_t(input_data.end - input_data.pos))); + return input_data.end - input_data.pos; + } + + void set_output_data(eosio::vm::span data) { + auto& output_data = derived().get_state().output_data; + output_data.clear(); + output_data.insert(output_data.end(), data.begin(), data.end()); + } + + template + static void register_callbacks() { + // todo: preconditions + Rft::template add<&Derived::get_input_data>("env", "get_input_data"); + Rft::template add<&Derived::set_output_data>("env", "set_output_data"); + } +}; + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/chaindb.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/chaindb.hpp new file mode 100644 index 00000000000..7ecb0d2a9b5 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/chaindb.hpp @@ -0,0 +1,252 @@ +#pragma once + +#include +#include +#include +#include + +namespace b1::rodeos { + +class iterator_cache { + private: + chain_kv::view& view; + + struct table_key { + uint64_t code = {}; + uint64_t table = {}; + uint64_t scope = {}; + + std::tuple order() const { return { code, table, scope }; } + friend bool operator<(const table_key& a, const table_key& b) { return a.order() < b.order(); } + }; + std::vector tables; + std::map table_to_index; + + struct row_key { + int32_t table_index = {}; + uint64_t key = {}; + + std::tuple order() const { return { table_index, key }; } + friend bool operator<(const row_key& a, const row_key& b) { return a.order() < b.order(); } + }; + + struct iterator { + int32_t table_index = {}; + uint64_t primary = {}; + std::vector value; + int32_t next = -1; + std::optional view_it; + }; + std::vector iterators; + std::vector end_iterators; + std::map key_to_iterator_index; + + int32_t get_table_index(table_key key) { + auto map_it = table_to_index.find(key); + if (map_it != table_to_index.end()) + return map_it->second; + if (!view.get(state_account.value, + chain_kv::to_slice(eosio::convert_to_key(std::make_tuple( + (uint8_t)0x01, eosio::name{ "contract.tab" }, + eosio::name{ "primary" }, key.code, key.table, key.scope))))) + return -1; + if (tables.size() != table_to_index.size() || tables.size() != end_iterators.size()) + throw std::runtime_error("internal error: tables.size() mismatch"); + auto result = tables.size(); + if (result > std::numeric_limits::max()) + throw std::runtime_error("too many open tables"); + tables.push_back(key); + table_to_index[key] = result; + end_iterators.push_back({}); + auto& end_it = end_iterators.back(); + end_it.table_index = result; + return result; + } + + int32_t get_iterator(row_key rk, chain_kv::view::iterator&& view_it) { + iterator* it; + int32_t result; + if (view_it.is_end()) { + // std::cout << "...end\n"; + it = &end_iterators[rk.table_index]; + result = index_to_end_iterator(rk.table_index); + } else { + auto map_it = key_to_iterator_index.find(rk); + if (map_it != key_to_iterator_index.end()) { + // std::cout << "...existing it (b)\n"; + it = &iterators[map_it->second]; + result = map_it->second; + } else { + // std::cout << "...new it\n"; + if (iterators.size() > std::numeric_limits::max()) + throw std::runtime_error("too many iterators"); + result = iterators.size(); + iterators.emplace_back(); + it = &iterators.back(); + eosio::input_stream stream{ view_it.get_kv()->value.data(), view_it.get_kv()->value.size() }; + auto row = std::get<0>(eosio::from_bin(stream)); + it->table_index = rk.table_index; + it->primary = row.primary_key; + it->value.insert(it->value.end(), row.value.pos, row.value.end); + } + } + if (!it->view_it) + it->view_it = std::move(view_it); + return result; + } + + // Precondition: std::numeric_limits::min() < ei < -1 + // Iterator of -1 is reserved for invalid iterators (i.e. when the appropriate table has not yet been created). + size_t end_iterator_to_index(int32_t ei) const { return (-ei - 2); } + // Precondition: indx < tables.size() <= std::numeric_limits::max() + int32_t index_to_end_iterator(size_t indx) const { return -(indx + 2); } + + public: + iterator_cache(chain_kv::view& view) : view{ view } {} + + size_t db_get_i64(int itr, char* buffer, uint32_t buffer_size) { + if (itr == -1) + throw std::runtime_error("dereference invalid iterator"); + if (itr < 0) + throw std::runtime_error("dereference end iterator"); + if (size_t(itr) >= iterators.size()) + throw std::runtime_error("dereference non-existing iterator"); + auto& it = iterators[itr]; + return legacy_copy_to_wasm(buffer, buffer_size, it.value.data(), it.value.size()); + } + + int db_next_i64(int itr, uint64_t& primary) { + if (itr == -1) + throw std::runtime_error("increment invalid iterator"); + if (itr < 0) + return -1; + if (size_t(itr) >= iterators.size()) + throw std::runtime_error("increment non-existing iterator"); + auto& it = iterators[itr]; + if (it.next >= 0) { + primary = iterators[it.next].primary; + return it.next; + } else if (it.next < -1) { + return it.next; + } + std::optional view_it = std::move(it.view_it); + it.view_it.reset(); + if (!view_it) { + // std::cout << "db_next_i64: db_view::iterator\n"; + const auto& table_key = tables[it.table_index]; + view_it = chain_kv::view::iterator{ + view, state_account.value, + chain_kv::to_slice(eosio::convert_to_key(std::make_tuple( // + (uint8_t)0x01, eosio::name{ "contract.row" }, eosio::name{ "primary" }, + table_key.code, table_key.table, table_key.scope))) + }; + view_it->lower_bound(eosio::convert_to_key(std::make_tuple( + (uint8_t)0x01, eosio::name{ "contract.row" }, eosio::name{ "primary" }, + table_key.code, table_key.table, table_key.scope, it.primary))); + } + ++*view_it; + if (view_it->is_end()) { + it.next = index_to_end_iterator(itr); + return it.next; + } else { + eosio::input_stream stream{ view_it->get_kv()->value.data(), view_it->get_kv()->value.size() }; + auto row = std::get<0>(eosio::from_bin(stream)); + primary = row.primary_key; + it.next = get_iterator({ it.table_index, primary }, std::move(*view_it)); + return it.next; + } + } + + int32_t lower_bound(uint64_t code, uint64_t scope, uint64_t table, uint64_t key) { + int32_t table_index = get_table_index({ code, table, scope }); + if (table_index < 0) { + // std::cout << "...no table\n"; + return -1; + } + row_key rk{ table_index, key }; + auto map_it = key_to_iterator_index.find(rk); + if (map_it != key_to_iterator_index.end()) { + // std::cout << "...existing it (a)\n"; + return map_it->second; + } + // std::cout << "lower_bound: db_view::iterator\n"; + chain_kv::view::iterator it{ view, state_account.value, + chain_kv::to_slice(eosio::convert_to_key(std::make_tuple( + (uint8_t)0x01, eosio::name{ "contract.row" }, + eosio::name{ "primary" }, code, table, scope))) }; + it.lower_bound(eosio::convert_to_key(std::make_tuple((uint8_t)0x01, eosio::name{ "contract.row" }, + eosio::name{ "primary" }, code, table, scope, key))); + return get_iterator(rk, std::move(it)); + } +}; // iterator_cache + +struct chaindb_state { + std::unique_ptr iterator_cache; +}; + +template +struct chaindb_callbacks { + Derived& derived() { return static_cast(*this); } + + iterator_cache& get_iterator_cache() { + auto& chaindb_state = derived().get_chaindb_state(); + if (!chaindb_state.iterator_cache) + chaindb_state.iterator_cache = std::make_unique(derived().get_db_view_state().kv_state.view); + return *chaindb_state.iterator_cache; + } + + int32_t db_store_i64(uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, legacy_span buffer) { + throw std::runtime_error("unimplemented: db_store_i64"); + } + + void db_update_i64(int32_t itr, uint64_t payer, legacy_span buffer) { + throw std::runtime_error("unimplemented: db_update_i64"); + } + + void db_remove_i64(int itr) { throw std::runtime_error("unimplemented: db_remove_i64"); } + + int32_t db_get_i64(int32_t itr, legacy_span buffer) { + return get_iterator_cache().db_get_i64(itr, buffer.data(), buffer.size()); + } + + int32_t db_next_i64(int32_t itr, legacy_ptr primary) { + return get_iterator_cache().db_next_i64(itr, *primary); + } + + int32_t db_previous_i64(int32_t itr, legacy_ptr primary) { + throw std::runtime_error("unimplemented: db_previous_i64"); + } + + int db_find_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) { + throw std::runtime_error("unimplemented: db_find_i64"); + } + + int db_lowerbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) { + return get_iterator_cache().lower_bound(code, scope, table, id); + } + + int db_upperbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) { + throw std::runtime_error("unimplemented: db_upperbound_i64"); + } + + int db_end_i64(uint64_t code, uint64_t scope, uint64_t table) { + throw std::runtime_error("unimplemented: db_end_i64"); + } + + template + static void register_callbacks() { + // todo: preconditions + Rft::template add<&Derived::db_store_i64>("env", "db_store_i64"); + Rft::template add<&Derived::db_update_i64>("env", "db_update_i64"); + Rft::template add<&Derived::db_remove_i64>("env", "db_remove_i64"); + Rft::template add<&Derived::db_get_i64>("env", "db_get_i64"); + Rft::template add<&Derived::db_next_i64>("env", "db_next_i64"); + Rft::template add<&Derived::db_previous_i64>("env", "db_previous_i64"); + Rft::template add<&Derived::db_find_i64>("env", "db_find_i64"); + Rft::template add<&Derived::db_lowerbound_i64>("env", "db_lowerbound_i64"); + Rft::template add<&Derived::db_upperbound_i64>("env", "db_upperbound_i64"); + Rft::template add<&Derived::db_end_i64>("env", "db_end_i64"); + } +}; + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/compiler_builtins.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/compiler_builtins.hpp new file mode 100644 index 00000000000..aa161a437eb --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/compiler_builtins.hpp @@ -0,0 +1,321 @@ +#pragma once + +#include +#include +#include + +namespace b1::rodeos { + +template +struct compiler_builtins_callbacks { + Derived& derived() { return static_cast(*this); } + + void __ashlti3(legacy_ptr<__int128> ret, uint64_t low, uint64_t high, uint32_t shift) { + if (shift >= 128) { + *ret = 0; + } else { + unsigned __int128 i = high; + i <<= 64; + i |= low; + i <<= shift; + *ret = (__int128)i; + } + } + + void __ashrti3(legacy_ptr<__int128> ret, uint64_t low, uint64_t high, uint32_t shift) { + // retain the signedness + *ret = high; + *ret <<= 64; + *ret |= low; + *ret >>= shift; + } + + void __lshlti3(legacy_ptr<__int128> ret, uint64_t low, uint64_t high, uint32_t shift) { + if (shift >= 128) { + *ret = 0; + } else { + unsigned __int128 i = high; + i <<= 64; + i |= low; + i <<= shift; + *ret = (__int128)i; + } + } + + void __lshrti3(legacy_ptr<__int128> ret, uint64_t low, uint64_t high, uint32_t shift) { + unsigned __int128 i = high; + i <<= 64; + i |= low; + i >>= shift; + *ret = (unsigned __int128)i; + } + + void __divti3(legacy_ptr<__int128> ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + __int128 lhs = ha; + __int128 rhs = hb; + + lhs <<= 64; + lhs |= la; + + rhs <<= 64; + rhs |= lb; + + if (rhs == 0) + throw std::runtime_error("divide by zero"); + + lhs /= rhs; + + *ret = lhs; + } + + void __udivti3(legacy_ptr ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + unsigned __int128 lhs = ha; + unsigned __int128 rhs = hb; + + lhs <<= 64; + lhs |= la; + + rhs <<= 64; + rhs |= lb; + + if (rhs == 0) + throw std::runtime_error("divide by zero"); + + lhs /= rhs; + *ret = lhs; + } + + void __modti3(legacy_ptr<__int128> ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + __int128 lhs = ha; + __int128 rhs = hb; + + lhs <<= 64; + lhs |= la; + + rhs <<= 64; + rhs |= lb; + + if (rhs == 0) + throw std::runtime_error("divide by zero"); + + lhs %= rhs; + *ret = lhs; + } + + void __umodti3(legacy_ptr ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + unsigned __int128 lhs = ha; + unsigned __int128 rhs = hb; + + lhs <<= 64; + lhs |= la; + + rhs <<= 64; + rhs |= lb; + + if (rhs == 0) + throw std::runtime_error("divide by zero"); + + lhs %= rhs; + *ret = lhs; + } + + void __multi3(legacy_ptr<__int128> ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + __int128 lhs = ha; + __int128 rhs = hb; + + lhs <<= 64; + lhs |= la; + + rhs <<= 64; + rhs |= lb; + + lhs *= rhs; + *ret = lhs; + } + + void __addtf3(legacy_ptr ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + float128_t a = { { la, ha } }; + float128_t b = { { lb, hb } }; + *ret = f128_add(a, b); + } + + void __subtf3(legacy_ptr ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + float128_t a = { { la, ha } }; + float128_t b = { { lb, hb } }; + *ret = f128_sub(a, b); + } + + void __multf3(legacy_ptr ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + float128_t a = { { la, ha } }; + float128_t b = { { lb, hb } }; + *ret = f128_mul(a, b); + } + + void __divtf3(legacy_ptr ret, uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + float128_t a = { { la, ha } }; + float128_t b = { { lb, hb } }; + *ret = f128_div(a, b); + } + + int __unordtf2(uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { + float128_t a = { { la, ha } }; + float128_t b = { { lb, hb } }; + if (f128_is_nan(a) || f128_is_nan(b)) + return 1; + return 0; + } + + int ___cmptf2(uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb, int return_value_if_nan) { + float128_t a = { { la, ha } }; + float128_t b = { { lb, hb } }; + if (__unordtf2(la, ha, lb, hb)) + return return_value_if_nan; + if (f128_lt(a, b)) + return -1; + if (f128_eq(a, b)) + return 0; + return 1; + } + + int __eqtf2(uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { return ___cmptf2(la, ha, lb, hb, 1); } + + int __netf2(uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { return ___cmptf2(la, ha, lb, hb, 1); } + + int __getf2(uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { return ___cmptf2(la, ha, lb, hb, -1); } + + int __gttf2(uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { return ___cmptf2(la, ha, lb, hb, 0); } + + int __letf2(uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { return ___cmptf2(la, ha, lb, hb, 1); } + + int __lttf2(uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { return ___cmptf2(la, ha, lb, hb, 0); } + + int __cmptf2(uint64_t la, uint64_t ha, uint64_t lb, uint64_t hb) { return ___cmptf2(la, ha, lb, hb, 1); } + + void __negtf2(legacy_ptr ret, uint64_t la, uint64_t ha) { *ret = { { la, (ha ^ (uint64_t)1 << 63) } }; } + + void __floatsitf(legacy_ptr ret, int32_t i) { *ret = i32_to_f128(i); } + + void __floatditf(legacy_ptr ret, uint64_t a) { *ret = i64_to_f128(a); } + + void __floatunsitf(legacy_ptr ret, uint32_t i) { *ret = ui32_to_f128(i); } + + void __floatunditf(legacy_ptr ret, uint64_t a) { *ret = ui64_to_f128(a); } + + double __floattidf(uint64_t l, uint64_t h) { + unsigned __int128 val = h; + val <<= 64; + val |= l; + return ___floattidf(*(__int128*)&val); + } + + double __floatuntidf(uint64_t l, uint64_t h) { + unsigned __int128 val = h; + val <<= 64; + val |= l; + return ___floatuntidf((unsigned __int128)val); + } + + double __floatsidf(int32_t i) { return from_softfloat64(i32_to_f64(i)); } + + void __extendsftf2(legacy_ptr ret, float f) { *ret = f32_to_f128(to_softfloat32(f)); } + + void __extenddftf2(legacy_ptr ret, double d) { *ret = f64_to_f128(to_softfloat64(d)); } + + void __fixtfti(legacy_ptr<__int128> ret, uint64_t l, uint64_t h) { + float128_t f = { { l, h } }; + *ret = ___fixtfti(f); + } + + int32_t __fixtfsi(uint64_t l, uint64_t h) { + float128_t f = { { l, h } }; + return f128_to_i32(f, 0, false); + } + + int64_t __fixtfdi(uint64_t l, uint64_t h) { + float128_t f = { { l, h } }; + return f128_to_i64(f, 0, false); + } + + void __fixunstfti(legacy_ptr ret, uint64_t l, uint64_t h) { + float128_t f = { { l, h } }; + *ret = ___fixunstfti(f); + } + + uint32_t __fixunstfsi(uint64_t l, uint64_t h) { + float128_t f = { { l, h } }; + return f128_to_ui32(f, 0, false); + } + + uint64_t __fixunstfdi(uint64_t l, uint64_t h) { + float128_t f = { { l, h } }; + return f128_to_ui64(f, 0, false); + } + + void __fixsfti(legacy_ptr<__int128> ret, float a) { *ret = ___fixsfti(to_softfloat32(a).v); } + + void __fixdfti(legacy_ptr<__int128> ret, double a) { *ret = ___fixdfti(to_softfloat64(a).v); } + + void __fixunssfti(legacy_ptr ret, float a) { *ret = ___fixunssfti(to_softfloat32(a).v); } + + void __fixunsdfti(legacy_ptr ret, double a) { *ret = ___fixunsdfti(to_softfloat64(a).v); } + + double __trunctfdf2(uint64_t l, uint64_t h) { + float128_t f = { { l, h } }; + return from_softfloat64(f128_to_f64(f)); + } + + float __trunctfsf2(uint64_t l, uint64_t h) { + float128_t f = { { l, h } }; + return from_softfloat32(f128_to_f32(f)); + } + + template + static void register_callbacks() { + // todo: preconditions + Rft::template add<&Derived::__ashlti3>("env", "__ashlti3"); + Rft::template add<&Derived::__ashrti3>("env", "__ashrti3"); + Rft::template add<&Derived::__lshlti3>("env", "__lshlti3"); + Rft::template add<&Derived::__lshrti3>("env", "__lshrti3"); + Rft::template add<&Derived::__divti3>("env", "__divti3"); + Rft::template add<&Derived::__udivti3>("env", "__udivti3"); + Rft::template add<&Derived::__modti3>("env", "__modti3"); + Rft::template add<&Derived::__umodti3>("env", "__umodti3"); + Rft::template add<&Derived::__multi3>("env", "__multi3"); + Rft::template add<&Derived::__addtf3>("env", "__addtf3"); + Rft::template add<&Derived::__subtf3>("env", "__subtf3"); + Rft::template add<&Derived::__multf3>("env", "__multf3"); + Rft::template add<&Derived::__divtf3>("env", "__divtf3"); + Rft::template add<&Derived::__eqtf2>("env", "__eqtf2"); + Rft::template add<&Derived::__netf2>("env", "__netf2"); + Rft::template add<&Derived::__getf2>("env", "__getf2"); + Rft::template add<&Derived::__gttf2>("env", "__gttf2"); + Rft::template add<&Derived::__lttf2>("env", "__lttf2"); + Rft::template add<&Derived::__letf2>("env", "__letf2"); + Rft::template add<&Derived::__cmptf2>("env", "__cmptf2"); + Rft::template add<&Derived::__unordtf2>("env", "__unordtf2"); + Rft::template add<&Derived::__negtf2>("env", "__negtf2"); + Rft::template add<&Derived::__floatsitf>("env", "__floatsitf"); + Rft::template add<&Derived::__floatunsitf>("env", "__floatunsitf"); + Rft::template add<&Derived::__floatditf>("env", "__floatditf"); + Rft::template add<&Derived::__floatunditf>("env", "__floatunditf"); + Rft::template add<&Derived::__floattidf>("env", "__floattidf"); + Rft::template add<&Derived::__floatuntidf>("env", "__floatuntidf"); + Rft::template add<&Derived::__floatsidf>("env", "__floatsidf"); + Rft::template add<&Derived::__extendsftf2>("env", "__extendsftf2"); + Rft::template add<&Derived::__extenddftf2>("env", "__extenddftf2"); + Rft::template add<&Derived::__fixtfti>("env", "__fixtfti"); + Rft::template add<&Derived::__fixtfdi>("env", "__fixtfdi"); + Rft::template add<&Derived::__fixtfsi>("env", "__fixtfsi"); + Rft::template add<&Derived::__fixunstfti>("env", "__fixunstfti"); + Rft::template add<&Derived::__fixunstfdi>("env", "__fixunstfdi"); + Rft::template add<&Derived::__fixunstfsi>("env", "__fixunstfsi"); + Rft::template add<&Derived::__fixsfti>("env", "__fixsfti"); + Rft::template add<&Derived::__fixdfti>("env", "__fixdfti"); + Rft::template add<&Derived::__fixunssfti>("env", "__fixunssfti"); + Rft::template add<&Derived::__fixunsdfti>("env", "__fixunsdfti"); + Rft::template add<&Derived::__trunctfdf2>("env", "__trunctfdf2"); + Rft::template add<&Derived::__trunctfsf2>("env", "__trunctfsf2"); + } +}; + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/console.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/console.hpp new file mode 100644 index 00000000000..9dda9f26a68 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/console.hpp @@ -0,0 +1,82 @@ +#pragma once + +#include +#include + +namespace b1::rodeos { + +struct console_state { + uint32_t max_console_size = 0; + std::string console = {}; +}; + +template +struct console_callbacks { + Derived& derived() { return static_cast(*this); } + + void append_console(const char* str, uint32_t len) { + auto& state = derived().get_state(); + state.console.append(str, std::min(size_t(len), state.max_console_size - state.console.size())); + } + + template + void append_console(const char (&str)[size]) { + return append_console(str, size - 1); // omit 0-termination + } + + void prints(null_terminated_ptr str) { append_console(str.data(), str.size()); } + void prints_l(legacy_span str) { append_console(str.data(), str.size()); } + + template + void print_numeric(T value) { + auto& state = derived().get_state(); + if (!state.max_console_size) + return; + auto s = std::to_string(value); + append_console(s.c_str(), s.size()); + } + + void printi(int64_t value) { print_numeric(value); } + void printui(uint64_t value) { print_numeric(value); } + void printi128(legacy_ptr p) { append_console("{printi128 unimplemented}"); } + void printui128(legacy_ptr p) { append_console("{printui128 unimplemented}"); } + void printsf(float value) { print_numeric(value); } + void printdf(double value) { print_numeric(value); } + void printqf(legacy_ptr) { append_console("{printqf unimplemented}"); } + + void printn(uint64_t value) { + auto& state = derived().get_state(); + if (!state.max_console_size) + return; + auto s = eosio::name{ value }.to_string(); + append_console(s.c_str(), s.size()); + } + + void printhex(legacy_span data) { + auto& state = derived().get_state(); + for (uint32_t i = 0; i < data.size() && state.console.size() + 2 < state.max_console_size; ++i) { + static const char hex_digits[] = "0123456789ABCDEF"; + uint8_t byte = data[i]; + state.console.push_back(hex_digits[byte >> 4]); + state.console.push_back(hex_digits[byte & 15]); + } + } + + template + static void register_callbacks() { + // todo: preconditions + Rft::template add<&Derived::prints>("env", "prints"); + Rft::template add<&Derived::prints_l>("env", "prints_l"); + Rft::template add<&Derived::printi>("env", "printi"); + Rft::template add<&Derived::printui>("env", "printui"); + Rft::template add<&Derived::printi128>("env", "printi128"); + Rft::template add<&Derived::printui128>("env", "printui128"); + Rft::template add<&Derived::printsf>("env", "printsf"); + Rft::template add<&Derived::printdf>("env", "printdf"); + Rft::template add<&Derived::printqf>("env", "printqf"); + Rft::template add<&Derived::printn>("env", "printn"); + Rft::template add<&Derived::printhex>("env", "printhex"); + } +}; // console_callbacks + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/filter.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/filter.hpp new file mode 100644 index 00000000000..fff307cafed --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/filter.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include + +namespace b1::rodeos { + +struct filter_callback_state { + std::function push_data; +}; + +template +struct filter_callbacks { + Derived& derived() { return static_cast(*this); } + + void push_data(eosio::vm::span data) { + derived().get_filter_callback_state().push_data(data.data(), data.size()); + } + + template + static void register_callbacks() { + // todo: preconditions + Rft::template add<&Derived::push_data>("env", "push_data"); + } +}; // query_callbacks + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/kv.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/kv.hpp new file mode 100644 index 00000000000..495e4ad959a --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/kv.hpp @@ -0,0 +1,427 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace b1::rodeos { + +inline constexpr eosio::name kvram_db_id{ "eosio.kvram" }; +inline constexpr eosio::name kvdisk_db_id{ "eosio.kvdisk" }; +inline constexpr eosio::name state_db_id{ "eosio.state" }; + +enum class kv_it_stat { + iterator_ok = 0, // Iterator is positioned at a key-value pair + iterator_erased = -1, // The key-value pair that the iterator used to be positioned at was erased + iterator_end = -2, // Iterator is out-of-bounds +}; + +struct kv_iterator_rocksdb { + uint32_t& num_iterators; + chain_kv::view& view; + uint64_t contract; + chain_kv::view::iterator kv_it; + + kv_iterator_rocksdb(uint32_t& num_iterators, chain_kv::view& view, uint64_t contract, const char* prefix, + uint32_t size) + : num_iterators(num_iterators), view{ view }, contract{ contract }, kv_it{ view, contract, { prefix, size } } { + ++num_iterators; + } + + ~kv_iterator_rocksdb() { --num_iterators; } + + bool is_kv_chainbase_context_iterator() const { return false; } + bool is_kv_rocksdb_context_iterator() const { return true; } + + kv_it_stat kv_it_status() { + if (kv_it.is_end()) + return kv_it_stat::iterator_end; + else if (kv_it.is_erased()) + return kv_it_stat::iterator_erased; + else + return kv_it_stat::iterator_ok; + } + + void fill_found(uint32_t* found_key_size, uint32_t* found_value_size) { + auto kv = kv_it.get_kv(); + if (kv) { + if (found_key_size) + *found_key_size = kv->key.size(); + if (found_value_size) + *found_value_size = kv->value.size(); + } else { + if (found_key_size) + *found_key_size = 0; + if (found_value_size) + *found_value_size = 0; + } + } + + int32_t kv_it_compare(const kv_iterator_rocksdb& rhs) { + eosio::check(rhs.is_kv_rocksdb_context_iterator(), "Incompatible key-value iterators"); + auto& r = static_cast(rhs); + eosio::check(&view == &r.view && contract == r.contract, "Incompatible key-value iterators"); + eosio::check(!kv_it.is_erased(), "Iterator to erased element"); + eosio::check(!r.kv_it.is_erased(), "Iterator to erased element"); + return compare(kv_it, r.kv_it); + } + + int32_t kv_it_key_compare(const char* key, uint32_t size) { + eosio::check(!kv_it.is_erased(), "Iterator to erased element"); + return chain_kv::compare_key(kv_it.get_kv(), chain_kv::key_value{ { key, size }, {} }); + } + + kv_it_stat kv_it_move_to_end() { + kv_it.move_to_end(); + return kv_it_stat::iterator_end; + } + + kv_it_stat kv_it_next(uint32_t* found_key_size = nullptr, uint32_t* found_value_size = nullptr) { + eosio::check(!kv_it.is_erased(), "Iterator to erased element"); + ++kv_it; + fill_found(found_key_size, found_value_size); + return kv_it_status(); + } + + kv_it_stat kv_it_prev(uint32_t* found_key_size = nullptr, uint32_t* found_value_size = nullptr) { + eosio::check(!kv_it.is_erased(), "Iterator to erased element"); + --kv_it; + fill_found(found_key_size, found_value_size); + return kv_it_status(); + } + + kv_it_stat kv_it_lower_bound(const char* key, uint32_t size, uint32_t* found_key_size = nullptr, + uint32_t* found_value_size = nullptr) { + kv_it.lower_bound(key, size); + fill_found(found_key_size, found_value_size); + return kv_it_status(); + } + + kv_it_stat kv_it_key(uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) { + eosio::check(!kv_it.is_erased(), "Iterator to erased element"); + + std::optional kv; + kv = kv_it.get_kv(); + + if (kv) { + if (offset < kv->key.size()) + memcpy(dest, kv->key.data() + offset, std::min((size_t)size, kv->key.size() - offset)); + actual_size = kv->key.size(); + return kv_it_stat::iterator_ok; + } else { + actual_size = 0; + return kv_it_stat::iterator_end; + } + } + + kv_it_stat kv_it_value(uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) { + eosio::check(!kv_it.is_erased(), "Iterator to erased element"); + + std::optional kv; + kv = kv_it.get_kv(); + + if (kv) { + if (offset < kv->value.size()) + memcpy(dest, kv->value.data() + offset, std::min((size_t)size, kv->value.size() - offset)); + actual_size = kv->value.size(); + return kv_it_stat::iterator_ok; + } else { + actual_size = 0; + return kv_it_stat::iterator_end; + } + } +}; // kv_iterator_rocksdb + +struct kv_database_config { + std::uint32_t max_key_size = 1024; + std::uint32_t max_value_size = 256 * 1024; // Large enough to hold most contracts + std::uint32_t max_iterators = 1024; +}; + +struct kv_context_rocksdb { + chain_kv::database& database; + chain_kv::write_session& write_session; + std::vector contract_kv_prefix; + eosio::name database_id; + chain_kv::view view; + bool enable_write = false; + bool bypass_receiver_check = false; + eosio::name receiver; + const kv_database_config& limits; + uint32_t num_iterators = 0; + std::shared_ptr> temp_data_buffer; + + kv_context_rocksdb(chain_kv::database& database, chain_kv::write_session& write_session, + std::vector contract_kv_prefix, eosio::name database_id, eosio::name receiver, + const kv_database_config& limits) + : database{ database }, write_session{ write_session }, contract_kv_prefix{ std::move(contract_kv_prefix) }, + database_id{ database_id }, view{ write_session, make_prefix() }, receiver{ receiver }, limits{ limits } {} + + std::vector make_prefix() { + std::vector prefix = contract_kv_prefix; + chain_kv::append_key(prefix, database_id.value); + return prefix; + } + + int64_t kv_erase(uint64_t contract, const char* key, uint32_t key_size) { + eosio::check(enable_write && (bypass_receiver_check || eosio::name{ contract } == receiver), + "Can not write to this key"); + temp_data_buffer = nullptr; + view.erase(contract, { key, key_size }); + return 0; + } + + int64_t kv_set(uint64_t contract, const char* key, uint32_t key_size, const char* value, uint32_t value_size) { + eosio::check(enable_write && (bypass_receiver_check || eosio::name{ contract } == receiver), + "Can not write to this key"); + eosio::check(key_size <= limits.max_key_size, "Key too large"); + eosio::check(value_size <= limits.max_value_size, "Value too large"); + temp_data_buffer = nullptr; + view.set(contract, { key, key_size }, { value, value_size }); + return 0; + } + + bool kv_get(uint64_t contract, const char* key, uint32_t key_size, uint32_t& value_size) { + temp_data_buffer = view.get(contract, { key, key_size }); + if (temp_data_buffer) { + value_size = temp_data_buffer->size(); + return true; + } else { + value_size = 0; + return false; + } + } + + uint32_t kv_get_data(uint32_t offset, char* data, uint32_t data_size) { + const char* temp = nullptr; + uint32_t temp_size = 0; + if (temp_data_buffer) { + temp = temp_data_buffer->data(); + temp_size = temp_data_buffer->size(); + } + if (offset < temp_size) + memcpy(data, temp + offset, std::min(data_size, temp_size - offset)); + return temp_size; + } + + std::unique_ptr kv_it_create(uint64_t contract, const char* prefix, uint32_t size) { + eosio::check(num_iterators < limits.max_iterators, "Too many iterators"); + return std::make_unique(num_iterators, view, contract, prefix, size); + } +}; // kv_context_rocksdb + +struct db_view_state { + eosio::name receiver; + chain_kv::database& database; + const kv_database_config limits; + const kv_database_config kv_state_limits{ 1024, std::numeric_limits::max() }; + kv_context_rocksdb kv_ram; + kv_context_rocksdb kv_disk; + kv_context_rocksdb kv_state; + std::vector> kv_iterators; + std::vector kv_destroyed_iterators; + + db_view_state(eosio::name receiver, chain_kv::database& database, chain_kv::write_session& write_session, + const std::vector& contract_kv_prefix) + : receiver{ receiver }, database{ database }, // + kv_ram{ database, write_session, contract_kv_prefix, kvram_db_id, receiver, limits }, + kv_disk{ database, write_session, contract_kv_prefix, kvdisk_db_id, receiver, limits }, + kv_state{ database, write_session, contract_kv_prefix, state_db_id, receiver, kv_state_limits }, + kv_iterators(1) {} + + void reset() { + eosio::check(kv_iterators.size() == kv_destroyed_iterators.size() + 1, "iterators are still alive"); + kv_iterators.resize(1); + kv_destroyed_iterators.clear(); + } +}; + +template +struct db_callbacks { + Derived& derived() { return static_cast(*this); } + + int64_t kv_erase(uint64_t db, uint64_t contract, eosio::vm::span key) { + return kv_get_db(db).kv_erase(contract, key.data(), key.size()); + } + + int64_t kv_set(uint64_t db, uint64_t contract, eosio::vm::span key, eosio::vm::span value) { + return kv_get_db(db).kv_set(contract, key.data(), key.size(), value.data(), value.size()); + } + + bool kv_get(uint64_t db, uint64_t contract, eosio::vm::span key, uint32_t* value_size) { + return kv_get_db(db).kv_get(contract, key.data(), key.size(), *value_size); + } + + uint32_t kv_get_data(uint64_t db, uint32_t offset, eosio::vm::span data) { + return kv_get_db(db).kv_get_data(offset, data.data(), data.size()); + } + + uint32_t kv_it_create(uint64_t db, uint64_t contract, eosio::vm::span prefix) { + auto& kdb = kv_get_db(db); + uint32_t itr; + if (!derived().get_db_view_state().kv_destroyed_iterators.empty()) { + itr = derived().get_db_view_state().kv_destroyed_iterators.back(); + derived().get_db_view_state().kv_destroyed_iterators.pop_back(); + } else { + // Sanity check in case the per-database limits are set poorly + eosio::check(derived().get_db_view_state().kv_iterators.size() <= 0xFFFFFFFFu, "Too many iterators"); + itr = derived().get_db_view_state().kv_iterators.size(); + derived().get_db_view_state().kv_iterators.emplace_back(); + } + derived().get_db_view_state().kv_iterators[itr] = kdb.kv_it_create(contract, prefix.data(), prefix.size()); + return itr; + } + + void kv_it_destroy(uint32_t itr) { + kv_check_iterator(itr); + derived().get_db_view_state().kv_destroyed_iterators.push_back(itr); + derived().get_db_view_state().kv_iterators[itr].reset(); + } + + int32_t kv_it_status(uint32_t itr) { + kv_check_iterator(itr); + return static_cast(derived().get_db_view_state().kv_iterators[itr]->kv_it_status()); + } + + int32_t kv_it_compare(uint32_t itr_a, uint32_t itr_b) { + kv_check_iterator(itr_a); + kv_check_iterator(itr_b); + return derived().get_db_view_state().kv_iterators[itr_a]->kv_it_compare( + *derived().get_db_view_state().kv_iterators[itr_b]); + } + + int32_t kv_it_key_compare(uint32_t itr, eosio::vm::span key) { + kv_check_iterator(itr); + return derived().get_db_view_state().kv_iterators[itr]->kv_it_key_compare(key.data(), key.size()); + } + + int32_t kv_it_move_to_end(uint32_t itr) { + kv_check_iterator(itr); + return static_cast(derived().get_db_view_state().kv_iterators[itr]->kv_it_move_to_end()); + } + + int32_t kv_it_next(uint32_t itr, uint32_t* found_key_size, uint32_t* found_value_size) { + kv_check_iterator(itr); + return static_cast( + derived().get_db_view_state().kv_iterators[itr]->kv_it_next(found_key_size, found_value_size)); + } + + int32_t kv_it_prev(uint32_t itr, uint32_t* found_key_size, uint32_t* found_value_size) { + kv_check_iterator(itr); + return static_cast( + derived().get_db_view_state().kv_iterators[itr]->kv_it_prev(found_key_size, found_value_size)); + } + + int32_t kv_it_lower_bound(uint32_t itr, eosio::vm::span key, uint32_t* found_key_size, + uint32_t* found_value_size) { + kv_check_iterator(itr); + return static_cast(derived().get_db_view_state().kv_iterators[itr]->kv_it_lower_bound( + key.data(), key.size(), found_key_size, found_value_size)); + } + + int32_t kv_it_key(uint32_t itr, uint32_t offset, eosio::vm::span dest, uint32_t* actual_size) { + kv_check_iterator(itr); + return static_cast( + derived().get_db_view_state().kv_iterators[itr]->kv_it_key(offset, dest.data(), dest.size(), *actual_size)); + } + + int32_t kv_it_value(uint32_t itr, uint32_t offset, eosio::vm::span dest, uint32_t* actual_size) { + kv_check_iterator(itr); + return static_cast(derived().get_db_view_state().kv_iterators[itr]->kv_it_value( + offset, dest.data(), dest.size(), *actual_size)); + } + + kv_context_rocksdb& kv_get_db(uint64_t db) { + if (db == kvram_db_id.value) + return derived().get_db_view_state().kv_ram; + else if (db == kvdisk_db_id.value) + return derived().get_db_view_state().kv_disk; + else if (db == state_db_id.value) + return derived().get_db_view_state().kv_state; + throw std::runtime_error("Bad key-value database ID"); + } + + void kv_check_iterator(uint32_t itr) { + eosio::check(itr < derived().get_db_view_state().kv_iterators.size() && + derived().get_db_view_state().kv_iterators[itr], + "Bad key-value iterator"); + } + + template + static void register_callbacks() { + // todo: preconditions + Rft::template add<&Derived::kv_erase>("env", "kv_erase"); + Rft::template add<&Derived::kv_set>("env", "kv_set"); + Rft::template add<&Derived::kv_get>("env", "kv_get"); + Rft::template add<&Derived::kv_get_data>("env", "kv_get_data"); + Rft::template add<&Derived::kv_it_create>("env", "kv_it_create"); + Rft::template add<&Derived::kv_it_destroy>("env", "kv_it_destroy"); + Rft::template add<&Derived::kv_it_status>("env", "kv_it_status"); + Rft::template add<&Derived::kv_it_compare>("env", "kv_it_compare"); + Rft::template add<&Derived::kv_it_key_compare>("env", "kv_it_key_compare"); + Rft::template add<&Derived::kv_it_move_to_end>("env", "kv_it_move_to_end"); + Rft::template add<&Derived::kv_it_next>("env", "kv_it_next"); + Rft::template add<&Derived::kv_it_prev>("env", "kv_it_prev"); + Rft::template add<&Derived::kv_it_lower_bound>("env", "kv_it_lower_bound"); + Rft::template add<&Derived::kv_it_key>("env", "kv_it_key"); + Rft::template add<&Derived::kv_it_value>("env", "kv_it_value"); + } +}; // db_callbacks + +class kv_environment : public db_callbacks { + public: + using base = db_callbacks; + db_view_state& state; + + kv_environment(db_view_state& state) : state{ state } {} + kv_environment(const kv_environment&) = default; + + auto& get_db_view_state() { return state; } + + int64_t kv_erase(uint64_t db, uint64_t contract, const char* key, uint32_t key_size) { + return base::kv_erase(db, contract, { key, key_size }); + } + + int64_t kv_set(uint64_t db, uint64_t contract, const char* key, uint32_t key_size, const char* value, + uint32_t value_size) { + return base::kv_set(db, contract, { key, key_size }, { value, value_size }); + } + + void kv_set(uint64_t db, uint64_t contract, const std::vector& k, const std::vector& v) { + base::kv_set(db, contract, { k.data(), k.size() }, { v.data(), v.size() }); + } + + bool kv_get(uint64_t db, uint64_t contract, const char* key, uint32_t key_size, uint32_t& value_size) { + return base::kv_get(db, contract, { key, key_size }, &value_size); + } + + uint32_t kv_get_data(uint64_t db, uint32_t offset, char* data, uint32_t data_size) { + return base::kv_get_data(db, offset, { data, data_size }); + } + + uint32_t kv_it_create(uint64_t db, uint64_t contract, const char* prefix, uint32_t prefix_size) { + return base::kv_it_create(db, contract, { prefix, prefix_size }); + } + + int32_t kv_it_key_compare(uint32_t itr, const char* key, uint32_t key_size) { + return base::kv_it_key_compare(itr, { key, key_size }); + } + + int32_t kv_it_lower_bound(uint32_t itr, const char* key, uint32_t key_size, uint32_t* found_key_size = nullptr, + uint32_t* found_value_size = nullptr) { + return base::kv_it_lower_bound(itr, { key, key_size }, found_key_size, found_value_size); + } + + int32_t kv_it_key(uint32_t itr, uint32_t offset, char* dest, uint32_t dest_size, uint32_t& actual_size) { + return base::kv_it_key(itr, offset, { dest, dest_size }, &actual_size); + } + + int32_t kv_it_value(uint32_t itr, uint32_t offset, char* dest, uint32_t dest_size, uint32_t& actual_size) { + return base::kv_it_value(itr, offset, { dest, dest_size }, &actual_size); + } +}; + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/memory.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/memory.hpp new file mode 100644 index 00000000000..d44c4df86bc --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/memory.hpp @@ -0,0 +1,54 @@ +#pragma once + +#include + +namespace b1::rodeos { + +template +struct memory_callbacks { + Derived& derived() { return static_cast(*this); } + + void* memcpy_impl(unvalidated_ptr dest, unvalidated_ptr src, wasm_size_t length) const { + volatile auto check_src = *((const char*)src + length - 1); + volatile auto check_dest = *((const char*)dest + length - 1); + volatile auto check_result = *(const char*)dest; + eosio::vm::ignore_unused_variable_warning(check_src, check_dest, check_result); + if ((size_t)(std::abs((ptrdiff_t)(char*)dest - (ptrdiff_t)(const char*)src)) < length) + throw std::runtime_error("memcpy can only accept non-aliasing pointers"); + return (char*)std::memcpy((char*)dest, (const char*)src, length); + } + + void* memmove_impl(unvalidated_ptr dest, unvalidated_ptr src, wasm_size_t length) const { + volatile auto check_src = *((const char*)src + length - 1); + volatile auto check_dest = *((const char*)dest + length - 1); + volatile auto check_result = *(const char*)dest; + eosio::vm::ignore_unused_variable_warning(check_src, check_dest, check_result); + return (char*)std::memmove((char*)dest, (const char*)src, length); + } + + int32_t memcmp_impl(unvalidated_ptr dest, unvalidated_ptr src, wasm_size_t length) const { + volatile auto check_src = *((const char*)src + length - 1); + volatile auto check_dest = *((const char*)dest + length - 1); + eosio::vm::ignore_unused_variable_warning(check_src, check_dest); + int32_t ret = std::memcmp((const char*)dest, (const char*)src, length); + return ret < 0 ? -1 : ret > 0 ? 1 : 0; + } + + void* memset_impl(unvalidated_ptr dest, int32_t value, wasm_size_t length) const { + volatile auto check_dest = *((const char*)dest + length - 1); + volatile auto check_result = *(const char*)dest; + eosio::vm::ignore_unused_variable_warning(check_dest, check_result); + return (char*)std::memset((char*)dest, value, length); + } + + template + static void register_callbacks() { + // todo: preconditions + Rft::template add<&Derived::memcpy_impl>("env", "memcpy"); + Rft::template add<&Derived::memmove_impl>("env", "memmove"); + Rft::template add<&Derived::memcmp_impl>("env", "memcmp"); + Rft::template add<&Derived::memset_impl>("env", "memset"); + } +}; // memory_callbacks + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/query.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/query.hpp new file mode 100644 index 00000000000..5b8070bda5a --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/query.hpp @@ -0,0 +1,45 @@ +#pragma once + +#include +#include + +namespace b1::rodeos { + +struct query_state { + uint32_t block_num; + std::optional block_info; +}; + +template +struct query_callbacks { + Derived& derived() { return static_cast(*this); } + + void load_block_info() { + auto& state = derived().get_state(); + if (state.block_info) + return; + auto info = get_state_row( + derived().get_db_view_state().kv_state.view, + std::make_tuple(eosio::name{ "block.info" }, eosio::name{ "primary" }, state.block_num)); + if (!info) + throw std::runtime_error("database is missing block.info for block " + std::to_string(state.block_num)); + state.block_info = info->second; + } + + int64_t current_time() { + load_block_info(); + return std::visit( + [](auto& b) { // + return b.timestamp.to_time_point().time_since_epoch().count(); + }, + *derived().get_state().block_info); + } + + template + static void register_callbacks() { + // todo: preconditions + Rft::template add<&Derived::current_time>("env", "current_time"); + } +}; // query_callbacks + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented.hpp new file mode 100644 index 00000000000..716d991e254 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented.hpp @@ -0,0 +1,245 @@ +#pragma once + +#include + +namespace b1::rodeos { + +template +struct unimplemented_callbacks { + template + T unimplemented(const char* name) { + throw std::runtime_error("wasm called " + std::string(name) + ", which is unimplemented"); + } + + // privileged_api + int is_feature_active(int64_t) { return unimplemented("is_feature_active"); } + void activate_feature(int64_t) { return unimplemented("activate_feature"); } + void get_resource_limits(int64_t, int, int, int) { return unimplemented("get_resource_limits"); } + void set_resource_limits(int64_t, int64_t, int64_t, int64_t) { return unimplemented("set_resource_limits"); } + int64_t set_proposed_producers(int, int) { return unimplemented("set_proposed_producers"); } + int get_blockchain_parameters_packed(int, int) { return unimplemented("get_blockchain_parameters_packed"); } + void set_blockchain_parameters_packed(int, int) { return unimplemented("set_blockchain_parameters_packed"); } + uint32_t get_parameters_packed(eosio::vm::span, eosio::vm::span) const { return unimplemented("get_parameters_packed"); } + void set_parameters_packed( eosio::vm::span ) { return unimplemented("set_parameters_packed"); } + + int is_privileged(int64_t) { return unimplemented("is_privileged"); } + void set_privileged(int64_t, int) { return unimplemented("set_privileged"); } + void preactivate_feature(int) { return unimplemented("preactivate_feature"); } + + // producer_api + int get_active_producers(int, int) { return unimplemented("get_active_producers"); } + +#define DB_SECONDARY_INDEX_METHODS_SIMPLE(IDX) \ + int db_##IDX##_store(int64_t, int64_t, int64_t, int64_t, int) { return unimplemented("db_" #IDX "_store"); } \ + void db_##IDX##_remove(int) { return unimplemented("db_" #IDX "_remove"); } \ + void db_##IDX##_update(int, int64_t, int) { return unimplemented("db_" #IDX "_update"); } \ + int db_##IDX##_find_primary(int64_t, int64_t, int64_t, int, int64_t) { \ + return unimplemented("db_" #IDX "_find_primary"); \ + } \ + int db_##IDX##_find_secondary(int64_t, int64_t, int64_t, int, int) { \ + return unimplemented("db_" #IDX "_find_secondary"); \ + } \ + int db_##IDX##_lowerbound(int64_t, int64_t, int64_t, int, int) { \ + return unimplemented("db_" #IDX "_lowerbound"); \ + } \ + int db_##IDX##_upperbound(int64_t, int64_t, int64_t, int, int) { \ + return unimplemented("db_" #IDX "_upperbound"); \ + } \ + int db_##IDX##_end(int64_t, int64_t, int64_t) { return unimplemented("db_" #IDX "_end"); } \ + int db_##IDX##_next(int, int) { return unimplemented("db_" #IDX "_next"); } \ + int db_##IDX##_previous(int, int) { return unimplemented("db_" #IDX "_previous"); } + +#define DB_SECONDARY_INDEX_METHODS_ARRAY(IDX) \ + int db_##IDX##_store(int64_t, int64_t, int64_t, int64_t, int, int) { \ + return unimplemented("db_" #IDX "_store"); \ + } \ + void db_##IDX##_remove(int) { return unimplemented("db_" #IDX "_remove"); } \ + void db_##IDX##_update(int, int64_t, int, int) { return unimplemented("db_" #IDX "_update"); } \ + int db_##IDX##_find_primary(int64_t, int64_t, int64_t, int, int, int64_t) { \ + return unimplemented("db_" #IDX "_find_primary"); \ + } \ + int db_##IDX##_find_secondary(int64_t, int64_t, int64_t, int, int, int) { \ + return unimplemented("db_" #IDX "_find_secondary"); \ + } \ + int db_##IDX##_lowerbound(int64_t, int64_t, int64_t, int, int, int) { \ + return unimplemented("db_" #IDX "_lowerbound"); \ + } \ + int db_##IDX##_upperbound(int64_t, int64_t, int64_t, int, int, int) { \ + return unimplemented("db_" #IDX "_upperbound"); \ + } \ + int db_##IDX##_end(int64_t, int64_t, int64_t) { return unimplemented("db_" #IDX "_end"); } \ + int db_##IDX##_next(int, int) { return unimplemented("db_" #IDX "_next"); } \ + int db_##IDX##_previous(int, int) { return unimplemented("db_" #IDX "_previous"); } + + // database_api + DB_SECONDARY_INDEX_METHODS_SIMPLE(idx64) + DB_SECONDARY_INDEX_METHODS_SIMPLE(idx128) + DB_SECONDARY_INDEX_METHODS_ARRAY(idx256) + DB_SECONDARY_INDEX_METHODS_SIMPLE(idx_double) + DB_SECONDARY_INDEX_METHODS_SIMPLE(idx_long_double) + +#undef DB_SECONDARY_INDEX_METHODS_SIMPLE +#undef DB_SECONDARY_INDEX_METHODS_ARRAY + + // crypto_api + void assert_recover_key(int, int, int, int, int) { return unimplemented("assert_recover_key"); } + int recover_key(int, int, int, int, int) { return unimplemented("recover_key"); } + void assert_sha256(int, int, int) { return unimplemented("assert_sha256"); } + void assert_sha1(int, int, int) { return unimplemented("assert_sha1"); } + void assert_sha512(int, int, int) { return unimplemented("assert_sha512"); } + void assert_ripemd160(int, int, int) { return unimplemented("assert_ripemd160"); } + void sha1(int, int, int) { return unimplemented("sha1"); } + void sha256(int, int, int) { return unimplemented("sha256"); } + void sha512(int, int, int) { return unimplemented("sha512"); } + void ripemd160(int, int, int) { return unimplemented("ripemd160"); } + + // permission_api + int check_transaction_authorization(int, int, int, int, int, int) { + return unimplemented("check_transaction_authorization"); + } + int check_permission_authorization(int64_t, int64_t, int, int, int, int, int64_t) { + return unimplemented("check_permission_authorization"); + } + int64_t get_permission_last_used(int64_t, int64_t) { return unimplemented("get_permission_last_used"); } + int64_t get_account_creation_time(int64_t) { return unimplemented("get_account_creation_time"); } + + // system_api + int64_t publication_time() { return unimplemented("publication_time"); } + int is_feature_activated(int) { return unimplemented("is_feature_activated"); } + int64_t get_sender() { return unimplemented("get_sender"); } + + // context_free_system_api + void eosio_assert_code(int, int64_t) { return unimplemented("eosio_assert_code"); } + void eosio_exit(int) { return unimplemented("eosio_exit"); } + + // authorization_api + void require_recipient(int64_t) { return unimplemented("require_recipient"); } + void require_auth(int64_t) { return unimplemented("require_auth"); } + void require_auth2(int64_t, int64_t) { return unimplemented("require_auth2"); } + int has_auth(int64_t) { return unimplemented("has_auth"); } + int is_account(int64_t) { return unimplemented("is_account"); } + + // context_free_transaction_api + int read_transaction(int, int) { return unimplemented("read_transaction"); } + int transaction_size() { return unimplemented("transaction_size"); } + int expiration() { return unimplemented("expiration"); } + int tapos_block_prefix() { return unimplemented("tapos_block_prefix"); } + int tapos_block_num() { return unimplemented("tapos_block_num"); } + int get_action(int, int, int, int) { return unimplemented("get_action"); } + + // transaction_api + void send_inline(int, int) { return unimplemented("send_inline"); } + void send_context_free_inline(int, int) { return unimplemented("send_context_free_inline"); } + void send_deferred(int, int64_t, int, int, int32_t) { return unimplemented("send_deferred"); } + int cancel_deferred(int) { return unimplemented("cancel_deferred"); } + + // context_free_api + int get_context_free_data(int, int, int) { return unimplemented("get_context_free_data"); } + + template + static void register_callbacks() { + // todo: preconditions + + // privileged_api + Rft::template add<&Derived::is_feature_active>("env", "is_feature_active"); + Rft::template add<&Derived::activate_feature>("env", "activate_feature"); + Rft::template add<&Derived::get_resource_limits>("env", "get_resource_limits"); + Rft::template add<&Derived::set_resource_limits>("env", "set_resource_limits"); + Rft::template add<&Derived::set_proposed_producers>("env", "set_proposed_producers"); + Rft::template add<&Derived::get_blockchain_parameters_packed>("env", "get_blockchain_parameters_packed"); + Rft::template add<&Derived::set_blockchain_parameters_packed>("env", "set_blockchain_parameters_packed"); + Rft::template add<&Derived::is_privileged>("env", "is_privileged"); + Rft::template add<&Derived::set_privileged>("env", "set_privileged"); + Rft::template add<&Derived::preactivate_feature>("env", "preactivate_feature"); + + // producer_api + Rft::template add<&Derived::get_active_producers>("env", "get_active_producers"); + +#define DB_SECONDARY_INDEX_METHODS_SIMPLE(IDX) \ + Rft::template add<&Derived::db_##IDX##_store>("env", "db_" #IDX "_store"); \ + Rft::template add<&Derived::db_##IDX##_remove>("env", "db_" #IDX "_remove"); \ + Rft::template add<&Derived::db_##IDX##_update>("env", "db_" #IDX "_update"); \ + Rft::template add<&Derived::db_##IDX##_find_primary>("env", "db_" #IDX "_find_primary"); \ + Rft::template add<&Derived::db_##IDX##_find_secondary>("env", "db_" #IDX "_find_secondary"); \ + Rft::template add<&Derived::db_##IDX##_lowerbound>("env", "db_" #IDX "_lowerbound"); \ + Rft::template add<&Derived::db_##IDX##_upperbound>("env", "db_" #IDX "_upperbound"); \ + Rft::template add<&Derived::db_##IDX##_end>("env", "db_" #IDX "_end"); \ + Rft::template add<&Derived::db_##IDX##_next>("env", "db_" #IDX "_next"); \ + Rft::template add<&Derived::db_##IDX##_previous>("env", "db_" #IDX "_previous"); + +#define DB_SECONDARY_INDEX_METHODS_ARRAY(IDX) \ + Rft::template add<&Derived::db_##IDX##_store>("env", "db_" #IDX "_store"); \ + Rft::template add<&Derived::db_##IDX##_remove>("env", "db_" #IDX "_remove"); \ + Rft::template add<&Derived::db_##IDX##_update>("env", "db_" #IDX "_update"); \ + Rft::template add<&Derived::db_##IDX##_find_primary>("env", "db_" #IDX "_find_primary"); \ + Rft::template add<&Derived::db_##IDX##_find_secondary>("env", "db_" #IDX "_find_secondary"); \ + Rft::template add<&Derived::db_##IDX##_lowerbound>("env", "db_" #IDX "_lowerbound"); \ + Rft::template add<&Derived::db_##IDX##_upperbound>("env", "db_" #IDX "_upperbound"); \ + Rft::template add<&Derived::db_##IDX##_end>("env", "db_" #IDX "_end"); \ + Rft::template add<&Derived::db_##IDX##_next>("env", "db_" #IDX "_next"); \ + Rft::template add<&Derived::db_##IDX##_previous>("env", "db_" #IDX "_previous"); + + // database_api + DB_SECONDARY_INDEX_METHODS_SIMPLE(idx64) + DB_SECONDARY_INDEX_METHODS_SIMPLE(idx128) + DB_SECONDARY_INDEX_METHODS_ARRAY(idx256) + DB_SECONDARY_INDEX_METHODS_SIMPLE(idx_double) + DB_SECONDARY_INDEX_METHODS_SIMPLE(idx_long_double) + +#undef DB_SECONDARY_INDEX_METHODS_SIMPLE +#undef DB_SECONDARY_INDEX_METHODS_ARRAY + + // crypto_api + Rft::template add<&Derived::assert_recover_key>("env", "assert_recover_key"); + Rft::template add<&Derived::recover_key>("env", "recover_key"); + Rft::template add<&Derived::assert_sha256>("env", "assert_sha256"); + Rft::template add<&Derived::assert_sha1>("env", "assert_sha1"); + Rft::template add<&Derived::assert_sha512>("env", "assert_sha512"); + Rft::template add<&Derived::assert_ripemd160>("env", "assert_ripemd160"); + Rft::template add<&Derived::sha1>("env", "sha1"); + Rft::template add<&Derived::sha256>("env", "sha256"); + Rft::template add<&Derived::sha512>("env", "sha512"); + Rft::template add<&Derived::ripemd160>("env", "ripemd160"); + + // permission_api + Rft::template add<&Derived::check_transaction_authorization>("env", "check_transaction_authorization"); + Rft::template add<&Derived::check_permission_authorization>("env", "check_permission_authorization"); + Rft::template add<&Derived::get_permission_last_used>("env", "get_permission_last_used"); + Rft::template add<&Derived::get_account_creation_time>("env", "get_account_creation_time"); + + // system_api + Rft::template add<&Derived::publication_time>("env", "publication_time"); + Rft::template add<&Derived::is_feature_activated>("env", "is_feature_activated"); + Rft::template add<&Derived::get_sender>("env", "get_sender"); + + // context_free_system_api + Rft::template add<&Derived::eosio_assert_code>("env", "eosio_assert_code"); + Rft::template add<&Derived::eosio_exit>("env", "eosio_exit"); + + // authorization_api + Rft::template add<&Derived::require_recipient>("env", "require_recipient"); + Rft::template add<&Derived::require_auth>("env", "require_auth"); + Rft::template add<&Derived::require_auth2>("env", "require_auth2"); + Rft::template add<&Derived::has_auth>("env", "has_auth"); + Rft::template add<&Derived::is_account>("env", "is_account"); + + // context_free_transaction_api + Rft::template add<&Derived::read_transaction>("env", "read_transaction"); + Rft::template add<&Derived::transaction_size>("env", "transaction_size"); + Rft::template add<&Derived::expiration>("env", "expiration"); + Rft::template add<&Derived::tapos_block_prefix>("env", "tapos_block_prefix"); + Rft::template add<&Derived::tapos_block_num>("env", "tapos_block_num"); + Rft::template add<&Derived::get_action>("env", "get_action"); + + // transaction_api + Rft::template add<&Derived::send_inline>("env", "send_inline"); + Rft::template add<&Derived::send_context_free_inline>("env", "send_context_free_inline"); + Rft::template add<&Derived::send_deferred>("env", "send_deferred"); + Rft::template add<&Derived::cancel_deferred>("env", "cancel_deferred"); + + // context_free_api + Rft::template add<&Derived::get_context_free_data>("env", "get_context_free_data"); + } // register_callbacks() +}; // unimplemented_callbacks + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented_filter.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented_filter.hpp new file mode 100644 index 00000000000..92143646f37 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/unimplemented_filter.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include + +namespace b1::rodeos { + +template +struct unimplemented_filter_callbacks { + template + T unimplemented(const char* name) { + throw std::runtime_error("wasm called " + std::string(name) + ", which is unimplemented"); + } + + // system_api + int64_t current_time() { return unimplemented("current_time"); } + + template + static void register_callbacks() { + // todo: preconditions + Rft::template add<&Derived::current_time>("env", "current_time"); + } +}; // unimplemented_filter_callbacks + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/vm_types.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/vm_types.hpp new file mode 100644 index 00000000000..1afae82a6f0 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/callbacks/vm_types.hpp @@ -0,0 +1,73 @@ +#pragma once + +#include + +#include + +namespace b1::rodeos { + +using wasm_size_t = eosio::vm::wasm_size_t; + +template +using legacy_ptr = eosio::vm::argument_proxy; + +template +using legacy_span = eosio::vm::argument_proxy, Align>; + +struct null_terminated_ptr : eosio::vm::span { + using base_type = eosio::vm::span; + null_terminated_ptr(const char* ptr) : base_type(ptr, strlen(ptr)) {} +}; + +// wrapper pointer type to keep the validations from being done +template +struct unvalidated_ptr { + static_assert(sizeof(T) == 1); // currently only going to support these for char and const char + operator T*() { return ptr; } + T* operator->() { return ptr; } + T& operator*() { return *ptr; } + T* ptr; +}; + +inline size_t legacy_copy_to_wasm(char* dest, size_t dest_size, const char* src, size_t src_size) { + if (dest_size == 0) + return src_size; + auto copy_size = std::min(dest_size, src_size); + memcpy(dest, src, copy_size); + return copy_size; +} + +template +struct type_converter : eosio::vm::type_converter { + using base_type = eosio::vm::type_converter; + using base_type::base_type; + using base_type::from_wasm; + + template + auto from_wasm(const void* ptr) const -> std::enable_if_t>, T> { + return { static_cast(ptr) }; + } + + template + auto from_wasm(void* ptr) const -> std::enable_if_t>, T> { + return { static_cast(ptr) }; + } + + template + auto from_wasm(void* ptr) const -> std::enable_if_t, eosio::vm::argument_proxy> { + this->template validate_pointer>(ptr, 1); + return { ptr }; + } + + EOS_VM_FROM_WASM(null_terminated_ptr, (const void* ptr)) { + this->validate_null_terminated_pointer(ptr); + return { static_cast(ptr) }; + } +}; // type_converter + +template +using registered_host_functions = + eosio::vm::registered_host_functions>; + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/constants.hpp b/libraries/rodeos/include/b1/rodeos/constants.hpp new file mode 100644 index 00000000000..094bed9587b --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/constants.hpp @@ -0,0 +1,13 @@ +#pragma once + +#include + +namespace b1::rodeos { + +// kv database which stores rodeos state, including a mirror of nodeos state +inline constexpr eosio::name state_database{ "eosio.state" }; + +// account within state_database which stores state +inline constexpr eosio::name state_account{ "state" }; + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/embedded_rodeos.h b/libraries/rodeos/include/b1/rodeos/embedded_rodeos.h new file mode 100644 index 00000000000..2fc58e399fe --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/embedded_rodeos.h @@ -0,0 +1,151 @@ +#pragma once + +#include "stdint.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct rodeos_error_s rodeos_error; +typedef struct rodeos_context_s rodeos_context; +typedef struct rodeos_db_partition_s rodeos_db_partition; +typedef struct rodeos_db_snapshot_s rodeos_db_snapshot; +typedef struct rodeos_filter_s rodeos_filter; +typedef struct rodeos_query_handler_s rodeos_query_handler; +typedef int rodeos_bool; + +// Create an error object. If multiple threads use an error object, then they must synchronize it. Returns NULL on +// failure. +rodeos_error* rodeos_create_error(); + +// Destroy an error object. This is a no-op if error == NULL. +void rodeos_destroy_error(rodeos_error* error); + +// Get last error in this object. Never returns NULL. The error object owns the returned string. +const char* rodeos_get_error(rodeos_error* error); + +// Create a context. Returns NULL on failure. +rodeos_context* rodeos_create(); + +// Destroy a context. It is undefined behavior if the context is used between threads without synchronization, or if any +// other objects currently exist for this context. This is a no-op if context == NULL. +void rodeos_destroy(rodeos_context* context); + +// Open database. num_threads is the target number of rocksdb background threads; use 0 for default. max_open_files is +// the max number of open rocksdb files; 0 to make this unlimited. +// +// It is undefined behavior if the context is used between threads without synchronization. Returns false on error. +rodeos_bool rodeos_open_db(rodeos_error* error, rodeos_context* context, const char* path, + rodeos_bool create_if_missing, int num_threads, int max_open_files); + +// Create or open a database partition. It is undefined behavior if more than 1 partition is opened for a given prefix, +// if any partitions have overlapping prefixes, or if the context is used between threads without synchronization. +// Returns NULL on failure. +rodeos_db_partition* rodeos_create_partition(rodeos_error* error, rodeos_context* context, const char* prefix, + uint32_t prefix_size); + +// Destroy a partition. It is undefined behavior if the partition is used between threads without synchronization. This +// is a no-op if partition == NULL. +void rodeos_destroy_partition(rodeos_db_partition* partition); + +// Create a database snapshot. Snapshots isolate changes from each other. All database reads and writes happen through +// snapshots. Snapshot objects may safely outlive partition objects. +// +// A single partition supports any number of simultaneous non-persistent snapshots, but only a single persistent +// snapshot at any time. persistent and non-persistent may coexist. Only persistent snapshots make permanent changes to +// the database. +// +// Each snapshot may be used by a different thread, even if they're created from a single partition. +// +// It is undefined behavior if more than 1 persistent snapshot exists on a partition, or if the partition is used +// between threads without synchronization. Returns NULL on failure. +rodeos_db_snapshot* rodeos_create_snapshot(rodeos_error* error, rodeos_db_partition* partition, rodeos_bool persistent); + +// Destroy a snapshot. It is undefined behavior if the snapshot is used between threads without synchronization. This is +// a no-op if snapshot == NULL. +void rodeos_destroy_snapshot(rodeos_db_snapshot* snapshot); + +// Refresh a snapshot so it may read recently-written database changes. This operation is invalid if the snapshot is +// persistent. It is undefined behavior if the snapshot is used between threads without synchronization. +rodeos_bool rodeos_refresh_snapshot(rodeos_error* error, rodeos_db_snapshot* snapshot); + +// Start writing a block. Aborts any block in progress and rolls back reversible blocks if needed. `data` must be the +// serialized `result` type defined by the state-history plugin's ABI. Currently only supports `get_blocks_result_v0`. +// It is undefined behavior if the snapshot is used between threads without synchronization. +rodeos_bool rodeos_start_block(rodeos_error* error, rodeos_db_snapshot* snapshot, const char* data, uint64_t size); + +// Finish writing a block. `data` must be the serialized `result` type defined by the state-history plugin's ABI. +// Currently only supports `get_blocks_result_v0`. If `force_write` is true, then the data will become immediately +// available to newly-created or newly-refreshed snapshots to read. If `force_write` is false, then the write may be +// delayed until a future rodeos_end_block call. It is undefined behavior if the snapshot is used between threads +// without synchronization. +rodeos_bool rodeos_end_block(rodeos_error* error, rodeos_db_snapshot* snapshot, const char* data, uint64_t size, + bool force_write); + +// Write block info. `data` must be the serialized `result` type defined by the state-history plugin's ABI. Currently +// only supports `get_blocks_result_v0`. If `rodeos_write_block_info` returns false, the snapshot will be in an +// inconsistent state; call `start_block` to abandon the current write and start another. It is undefined behavior if +// the snapshot is used between threads without synchronization. +rodeos_bool rodeos_write_block_info(rodeos_error* error, rodeos_db_snapshot* snapshot, const char* data, uint64_t size); + +// Write state-history deltas to a block. `data` must be the serialized `result` type defined by the state-history +// plugin's ABI. Currently only supports `get_blocks_result_v0`. If `shutdown` isn't null, then `rodeos_write_deltas` +// may call it during long operations. If `shutdown` returns true, then `rodeos_write_deltas` abandons the writes. If +// `rodeos_write_deltas` returns false, the snapshot will be in an inconsistent state; call `start_block` to abandon the +// current write and start another. It is undefined behavior if the snapshot is used between threads without +// synchronization. +rodeos_bool rodeos_write_deltas(rodeos_error* error, rodeos_db_snapshot* snapshot, const char* data, uint64_t size, + rodeos_bool (*shutdown)(void*), void* shutdown_arg); + +// Create a filter. Returns NULL on failure. +rodeos_filter* rodeos_create_filter(rodeos_error* error, uint64_t name, const char* wasm_filename); + +// Destroy a filter. It is undefined behavior if the filter is used between threads without synchronization. This is a +// no-op if filter == NULL. +void rodeos_destroy_filter(rodeos_filter* filter); + +// Run filter. data must be the serialized `result` type defined by the state-history plugin's ABI. The `push_data` +// callback receives data from the filter's `push_data` intrinsic. If `push_data` is null, then the intrinsic is a +// no-op. +// +// If `rodeos_run_filter` returns false, the snapshot will be in an inconsistent state; call `start_block` to abandon +// the current write and start another. It is undefined behavior if `snapshot` or `filter` is used between threads +// without synchronization. +rodeos_bool rodeos_run_filter(rodeos_error* error, rodeos_db_snapshot* snapshot, rodeos_filter* filter, + const char* data, uint64_t size, + rodeos_bool (*push_data)(void* arg, const char* data, uint64_t size), + void* push_data_arg); + +// Create a query handler. This object manages pools of resources for running queries simultaneously. +// +// Query handlers may safely outlive partition objects. It is undefined behavior if the partition is used between +// threads without synchronization. Returns NULL on failure. +// +// TODO: remove partition arg; redundant with snapshot in rodeos_query_transaction +rodeos_query_handler* rodeos_create_query_handler(rodeos_error* error, rodeos_db_partition* partition, + uint32_t max_console_size, uint32_t wasm_cache_size, + uint64_t max_exec_time_ms, const char* contract_dir); + +// Destroy a query handler. It is undefined behavior if the handler is used between threads without synchronization. +// This is a no-op if handler == NULL. +void rodeos_destroy_query_handler(rodeos_query_handler* handler); + +// Run a query. data is a serialized ship_protocol::packed_transaction. Returns false on serious error and sets *result +// to NULL and *result_size to 0. Otherwise, sets *result and *result_size to memory containing a serialized +// ship_protocol::transaction_trace. If the query failed, the error result will be in the transaction trace. Caller must +// use rodeos_free_result to free the memory. +// +// It is safe to use the same handler from multiple threads if: +// * The return from rodeos_create_query_handler happens-before any calls to rodeos_query_transaction +// * The return from all rodeos_query_transaction calls happens-before the call to rodeos_destroy_query_handler +// +// It is undefined behavior if `snapshot` is used between threads without synchronization. +rodeos_bool rodeos_query_transaction(rodeos_error* error, rodeos_query_handler* handler, rodeos_db_snapshot* snapshot, + const char* data, uint64_t size, char** result, uint64_t* result_size); + +// Frees memory from rodeos_query_transaction. Does nothing if result == NULL. +void rodeos_free_result(char* result); + +#ifdef __cplusplus +} +#endif diff --git a/libraries/rodeos/include/b1/rodeos/embedded_rodeos.hpp b/libraries/rodeos/include/b1/rodeos/embedded_rodeos.hpp new file mode 100644 index 00000000000..407b2513d3f --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/embedded_rodeos.hpp @@ -0,0 +1,198 @@ +#pragma once + +#include +#include +#include + +namespace b1::embedded_rodeos { + +struct error { + rodeos_error* obj; + + error() : obj{ rodeos_create_error() } { + if (!obj) + throw std::bad_alloc{}; + } + + error(const error&) = delete; + + ~error() { rodeos_destroy_error(obj); } + + error& operator=(const error&) = delete; + + operator rodeos_error*() { return obj; } + + const char* msg() { return rodeos_get_error(obj); } + + template + auto check(F f) -> decltype(f()) { + auto result = f(); + if (!result) + throw std::runtime_error(msg()); + return result; + } +}; + +struct context { + struct error error; + rodeos_context* obj; + + context() : obj{ rodeos_create() } { + if (!obj) + throw std::bad_alloc{}; + } + + context(const context&) = delete; + + ~context() { rodeos_destroy(obj); } + + context& operator=(const context&) = delete; + + operator rodeos_context*() { return obj; } + + void open_db(const char* path, bool create_if_missing, int num_threads = 0, int max_open_files = 0) { + error.check([&] { return rodeos_open_db(error, obj, path, create_if_missing, num_threads, max_open_files); }); + } +}; + +struct partition { + struct error error; + rodeos_db_partition* obj; + + partition(rodeos_context* context, const char* prefix, uint32_t prefix_size) { + obj = error.check([&] { return rodeos_create_partition(error, context, prefix, prefix_size); }); + } + + partition(const partition&) = delete; + + ~partition() { rodeos_destroy_partition(obj); } + + partition& operator=(const partition&) = delete; + + operator rodeos_db_partition*() { return obj; } +}; + +struct snapshot { + struct error error; + rodeos_db_snapshot* obj; + + snapshot(rodeos_db_partition* partition, bool persistent) { + obj = error.check([&] { return rodeos_create_snapshot(error, partition, persistent); }); + } + + snapshot(const snapshot&) = delete; + + ~snapshot() { rodeos_destroy_snapshot(obj); } + + snapshot& operator=(const snapshot&) = delete; + + operator rodeos_db_snapshot*() { return obj; } + + void refresh() { + error.check([&] { return rodeos_refresh_snapshot(error, obj); }); + } + + void start_block(const char* data, uint64_t size) { + error.check([&] { return rodeos_start_block(error, obj, data, size); }); + } + + void end_block(const char* data, uint64_t size, bool force_write) { + error.check([&] { return rodeos_end_block(error, obj, data, size, force_write); }); + } + + void write_block_info(const char* data, uint64_t size) { + error.check([&] { return rodeos_write_block_info(error, obj, data, size); }); + } + + template + void write_deltas(const char* data, uint64_t size, F shutdown) { + error.check([&] { + return rodeos_write_deltas( + error, obj, data, size, [](void* f) -> rodeos_bool { return (*static_cast(f))(); }, &shutdown); + }); + } +}; + +struct filter { + struct error error; + rodeos_filter* obj; + + filter(uint64_t name, const char* wasm_filename) { + obj = error.check([&] { return rodeos_create_filter(error, name, wasm_filename); }); + } + + filter(const filter&) = delete; + + ~filter() { rodeos_destroy_filter(obj); } + + filter& operator=(const filter&) = delete; + + operator rodeos_filter*() { return obj; } + + void run(rodeos_db_snapshot* snapshot, const char* data, uint64_t size) { + error.check([&] { return rodeos_run_filter(error, snapshot, obj, data, size, nullptr, nullptr); }); + } + + template + void run(rodeos_db_snapshot* snapshot, const char* data, uint64_t size, F push_data) { + error.check([&] { + return rodeos_run_filter( + error, snapshot, obj, data, size, + [](void* arg, const char* data, uint64_t size) -> rodeos_bool { + try { + return (*reinterpret_cast(arg))(data, size); + } catch (...) { return false; } + }, + &push_data); + }); + } +}; + +struct result { + char* data = {}; + uint64_t size = {}; + + result() = default; + result(const result&) = delete; + result(result&& src) { *this = std::move(src); } + ~result() { rodeos_free_result(data); } + + result& operator=(const result& src) = delete; + + result& operator=(result&& src) { + data = src.data; + size = src.size; + src.data = nullptr; + src.size = 0; + return *this; + } +}; + +struct query_handler { + struct error error; + rodeos_query_handler* obj; + + query_handler(rodeos_db_partition* partition, uint32_t max_console_size, uint32_t wasm_cache_size, + uint64_t max_exec_time_ms, const char* contract_dir) { + obj = error.check([&] { + return rodeos_create_query_handler(error, partition, max_console_size, wasm_cache_size, max_exec_time_ms, + contract_dir); + }); + } + + query_handler(const query_handler&) = delete; + + ~query_handler() { rodeos_destroy_query_handler(obj); } + + query_handler& operator=(const query_handler&) = delete; + + operator rodeos_query_handler*() { return obj; } + + result query_transaction(rodeos_db_snapshot* snapshot, const char* data, uint64_t size) { + result r; + error.check([&] { return rodeos_query_transaction(error, obj, snapshot, data, size, &r.data, &r.size); }); + return r; + } +}; + +} // namespace b1::embedded_rodeos diff --git a/libraries/rodeos/include/b1/rodeos/filter.hpp b/libraries/rodeos/include/b1/rodeos/filter.hpp new file mode 100644 index 00000000000..8b4b405d288 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/filter.hpp @@ -0,0 +1,61 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +// todo: configure limits +// todo: timeout +namespace b1::rodeos::filter { + +struct callbacks; +using rhf_t = registered_host_functions; +using backend_t = eosio::vm::backend; + +struct filter_state : b1::rodeos::data_state, b1::rodeos::console_state, b1::rodeos::filter_callback_state { + eosio::vm::wasm_allocator wa = {}; +}; + +struct callbacks : b1::rodeos::chaindb_callbacks, + b1::rodeos::compiler_builtins_callbacks, + b1::rodeos::console_callbacks, + b1::rodeos::context_free_system_callbacks, + b1::rodeos::data_callbacks, + b1::rodeos::db_callbacks, + b1::rodeos::filter_callbacks, + b1::rodeos::memory_callbacks, + b1::rodeos::unimplemented_callbacks, + b1::rodeos::unimplemented_filter_callbacks { + filter::filter_state& filter_state; + b1::rodeos::chaindb_state& chaindb_state; + b1::rodeos::db_view_state& db_view_state; + + callbacks(filter::filter_state& filter_state, b1::rodeos::chaindb_state& chaindb_state, + b1::rodeos::db_view_state& db_view_state) + : filter_state{ filter_state }, chaindb_state{ chaindb_state }, db_view_state{ db_view_state } {} + + auto& get_state() { return filter_state; } + auto& get_filter_callback_state() { return filter_state; } + auto& get_chaindb_state() { return chaindb_state; } + auto& get_db_view_state() { return db_view_state; } +}; + +inline void register_callbacks() { + b1::rodeos::chaindb_callbacks::register_callbacks(); + b1::rodeos::compiler_builtins_callbacks::register_callbacks(); + b1::rodeos::console_callbacks::register_callbacks(); + b1::rodeos::context_free_system_callbacks::register_callbacks(); + b1::rodeos::data_callbacks::register_callbacks(); + b1::rodeos::db_callbacks::register_callbacks(); + b1::rodeos::filter_callbacks::register_callbacks(); + b1::rodeos::memory_callbacks::register_callbacks(); + b1::rodeos::unimplemented_callbacks::register_callbacks(); + b1::rodeos::unimplemented_filter_callbacks::register_callbacks(); +} + +} // namespace b1::rodeos::filter diff --git a/libraries/rodeos/include/b1/rodeos/get_state_row.hpp b/libraries/rodeos/include/b1/rodeos/get_state_row.hpp new file mode 100644 index 00000000000..ff969135026 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/get_state_row.hpp @@ -0,0 +1,55 @@ +#pragma once + +#include +#include + +namespace b1::rodeos { + +template +std::optional, T>> get_state_row(chain_kv::view& view, const K& key) { + std::optional, T>> result; + result.emplace(); + result->first = + view.get(state_account.value, + chain_kv::to_slice(eosio::convert_to_key(std::make_tuple((uint8_t)0x01, key)))); + if (!result->first) { + result.reset(); + return result; + } + + eosio::input_stream stream{ *result->first }; + try { + from_bin(result->second, stream); + } catch(std::exception& e) { + throw std::runtime_error("An error occurred deserializing state: " + std::string(e.what())); + } + return result; +} + +template +std::optional, T>> get_state_row_secondary(chain_kv::view& view, + const K& key) { + std::optional, T>> result; + auto pk = + view.get(state_account.value, + chain_kv::to_slice(eosio::convert_to_key(std::make_tuple((uint8_t)0x01, key)))); + if (!pk) + return result; + + result.emplace(); + result->first = view.get(state_account.value, chain_kv::to_slice(*pk)); + if (!result->first) { + result.reset(); + return result; + } + + eosio::input_stream stream{ *result->first }; + try { + from_bin(result->second, stream); + } catch(std::exception& e) { + throw std::runtime_error("An error occurred deserializing state: " + std::string(e.what())); + } + return result; +} + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/rodeos.hpp b/libraries/rodeos/include/b1/rodeos/rodeos.hpp new file mode 100644 index 00000000000..6188c8daea0 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/rodeos.hpp @@ -0,0 +1,93 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace b1::rodeos { + +static constexpr char undo_prefix_byte = 0x01; +static constexpr char contract_kv_prefix_byte = 0x02; + +struct rodeos_context { + std::shared_ptr db; +}; + +struct rodeos_db_partition { + const std::shared_ptr db; + const std::vector undo_prefix; + const std::vector contract_kv_prefix; + + // todo: move rocksdb::ManagedSnapshot to here to prevent optimization in cloner from + // defeating non-persistent snapshots. + + rodeos_db_partition(std::shared_ptr db, const std::vector& prefix) + : db{ std::move(db) }, // + undo_prefix{ [&] { + auto x = prefix; + x.push_back(undo_prefix_byte); + return x; + }() }, + contract_kv_prefix{ [&] { + auto x = prefix; + x.push_back(contract_kv_prefix_byte); + return x; + }() } {} +}; + +struct rodeos_db_snapshot { + std::shared_ptr partition = {}; + std::shared_ptr db = {}; + std::optional undo_stack = {}; // only if persistent + std::optional snap = {}; // only if !persistent + std::optional write_session = {}; + eosio::checksum256 chain_id = {}; + uint32_t head = 0; + eosio::checksum256 head_id = {}; + uint32_t irreversible = 0; + eosio::checksum256 irreversible_id = {}; + uint32_t first = 0; + std::optional writing_block = {}; + + rodeos_db_snapshot(std::shared_ptr partition, bool persistent); + + void refresh(); + void end_write(bool write_fill); + void start_block(const eosio::ship_protocol::get_blocks_result_base& result); + void end_block(const eosio::ship_protocol::get_blocks_result_base& result, bool force_write); + void check_write(const eosio::ship_protocol::get_blocks_result_base& result); + void write_block_info(const eosio::ship_protocol::get_blocks_result_v0& result); + void write_block_info(const eosio::ship_protocol::get_blocks_result_v1& result); + void write_deltas(const eosio::ship_protocol::get_blocks_result_v0& result, std::function shutdown); + void write_deltas(const eosio::ship_protocol::get_blocks_result_v1& result, std::function shutdown); + + private: + void write_block_info(uint32_t block_num, const eosio::checksum256& id, + const eosio::ship_protocol::signed_block_header& block); + void write_deltas(uint32_t block_num, eosio::opaque> deltas, std::function shutdown); + void write_fill_status(); +}; + +struct rodeos_filter { + eosio::name name = {}; + std::unique_ptr backend = {}; + std::unique_ptr filter_state = {}; + + rodeos_filter(eosio::name name, const std::string& wasm_filename); + + void process(rodeos_db_snapshot& snapshot, const eosio::ship_protocol::get_blocks_result_base& result, + eosio::input_stream bin, const std::function& push_data); +}; + +struct rodeos_query_handler { + std::shared_ptr partition; + const std::shared_ptr shared_state; + wasm_ql::thread_state_cache state_cache; + + rodeos_query_handler(std::shared_ptr partition, + std::shared_ptr shared_state); + rodeos_query_handler(const rodeos_query_handler&) = delete; +}; + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp b/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp new file mode 100644 index 00000000000..a6b4e5b10d2 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp @@ -0,0 +1,267 @@ +#pragma once + +#include +#include +#include +#include + +namespace eosio { +using b1::rodeos::kv_environment; +} + +#include + +namespace b1::rodeos { + +using account = eosio::ship_protocol::account; +using account_metadata = eosio::ship_protocol::account_metadata; +using code = eosio::ship_protocol::code; +using contract_index128 = eosio::ship_protocol::contract_index128; +using contract_index64 = eosio::ship_protocol::contract_index64; +using contract_row = eosio::ship_protocol::contract_row; +using contract_table = eosio::ship_protocol::contract_table; +using global_property = eosio::ship_protocol::global_property; +using key_value = eosio::ship_protocol::key_value; +using key_value_v0 = eosio::ship_protocol::key_value_v0; +using producer_schedule = eosio::ship_protocol::producer_schedule; +using table_delta = eosio::ship_protocol::table_delta; +using table_delta_v1 = eosio::ship_protocol::table_delta_v1; + +struct fill_status_v0 { + eosio::checksum256 chain_id = {}; + uint32_t head = {}; + eosio::checksum256 head_id = {}; + uint32_t irreversible = {}; + eosio::checksum256 irreversible_id = {}; + uint32_t first = {}; +}; + +EOSIO_REFLECT(fill_status_v0, chain_id, head, head_id, irreversible, irreversible_id, first) + +using fill_status = std::variant; + +inline bool operator==(const fill_status_v0& a, fill_status_v0& b) { + return std::tie(a.head, a.head_id, a.irreversible, a.irreversible_id, a.first) == + std::tie(b.head, b.head_id, b.irreversible, b.irreversible_id, b.first); +} + +inline bool operator!=(const fill_status_v0& a, fill_status_v0& b) { return !(a == b); } + +using fill_status_sing = eosio::kv_singleton; + +struct block_info_v0 { + uint32_t num = {}; + eosio::checksum256 id = {}; + eosio::block_timestamp timestamp = {}; + eosio::name producer = {}; + uint16_t confirmed = {}; + eosio::checksum256 previous = {}; + eosio::checksum256 transaction_mroot = {}; + eosio::checksum256 action_mroot = {}; + uint32_t schedule_version = {}; + std::optional new_producers = {}; + eosio::signature producer_signature = {}; +}; + +EOSIO_REFLECT(block_info_v0, num, id, timestamp, producer, confirmed, previous, transaction_mroot, action_mroot, + schedule_version, new_producers, producer_signature) + +using block_info = std::variant; + +// todo: move out of "state"? +struct block_info_kv : eosio::kv_table { + index primary_index{ eosio::name{ "primary" }, [](const auto& var) { + return std::visit([](const auto& obj) { return obj.num; }, *var); + } }; + + index id_index{ eosio::name{ "id" }, [](const auto& var) { + return std::visit([](const auto& obj) { return obj.id; }, *var); + } }; + + block_info_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { + init(state_account, eosio::name{ "block.info" }, state_database, primary_index, id_index); + } +}; + +struct global_property_kv : eosio::kv_table { + index> primary_index{ eosio::name{ "primary" }, + [](const auto& var) { return std::vector{}; } }; + + global_property_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { + init(state_account, eosio::name{ "global.prop" }, state_database, primary_index); + } +}; + +struct account_kv : eosio::kv_table { + index primary_index{ eosio::name{ "primary" }, [](const auto& var) { + return std::visit([](const auto& obj) { return obj.name; }, *var); + } }; + + account_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { + init(state_account, eosio::name{ "account" }, state_database, primary_index); + } +}; + +struct account_metadata_kv : eosio::kv_table { + index primary_index{ eosio::name{ "primary" }, [](const auto& var) { + return std::visit([](const auto& obj) { return obj.name; }, *var); + } }; + + account_metadata_kv(eosio::kv_environment environment) + : eosio::kv_table{ std::move(environment) } { + init(state_account, eosio::name{ "account.meta" }, state_database, primary_index); + } +}; + +struct code_kv : eosio::kv_table { + index> primary_index{ + eosio::name{ "primary" }, + [](const auto& var) { + return std::visit([](const auto& obj) { return std::tie(obj.vm_type, obj.vm_version, obj.code_hash); }, *var); + } + }; + + code_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { + init(state_account, eosio::name{ "code" }, state_database, primary_index); + } +}; + +struct contract_table_kv : eosio::kv_table { + index> primary_index{ + eosio::name{ "primary" }, + [](const auto& var) { + return std::visit([](const auto& obj) { return std::tie(obj.code, obj.table, obj.scope); }, *var); + } + }; + + contract_table_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { + init(state_account, eosio::name{ "contract.tab" }, state_database, primary_index); + } +}; + +struct contract_row_kv : eosio::kv_table { + using PT = typename std::tuple; + index primary_index{ eosio::name{ "primary" }, [](const auto& var) { + return std::visit( + [](const auto& obj) { + return std::tie(obj.code, obj.table, obj.scope, obj.primary_key); + }, + *var); + } }; + + contract_row_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { + init(state_account, eosio::name{ "contract.row" }, state_database, primary_index); + } +}; + +struct contract_index64_kv : eosio::kv_table { + using PT = typename std::tuple; + index primary_index{ eosio::name{ "primary" }, [](const auto& var) { + return std::visit( + [](const auto& obj) { + return std::tie(obj.code, obj.table, obj.scope, obj.primary_key); + }, + *var); + } }; + using ST = typename std::tuple; + index secondary_index{ eosio::name{ "secondary" }, [](const auto& var) { + return std::visit( + [](const auto& obj) { + return std::tie(obj.code, obj.table, obj.scope, obj.secondary_key, + obj.primary_key); + }, + *var); + } }; + + contract_index64_kv(eosio::kv_environment environment) + : eosio::kv_table{ std::move(environment) } { + init(state_account, eosio::name{ "contract.i1" }, state_database, primary_index, secondary_index); + } +}; + +struct contract_index128_kv : eosio::kv_table { + using PT = typename std::tuple; + index primary_index{ eosio::name{ "primary" }, [](const auto& var) { + return std::visit( + [](const auto& obj) { + return std::tie(obj.code, obj.table, obj.scope, obj.primary_key); + }, + *var); + } }; + using ST = typename std::tuple; + index secondary_index{ eosio::name{ "secondary" }, [](const auto& var) { + return std::visit( + [](const auto& obj) { + return std::tie(obj.code, obj.table, obj.scope, obj.secondary_key, + obj.primary_key); + }, + *var); + } }; + + contract_index128_kv(eosio::kv_environment environment) + : eosio::kv_table{ std::move(environment) } { + init(state_account, eosio::name{ "contract.i2" }, state_database, primary_index, secondary_index); + } +}; + +template +void store_delta_typed(eosio::kv_environment environment, D& delta, bool bypass_preexist_check, F f) { + Table table{ environment }; + for (auto& row : delta.rows) { + f(); + auto obj = eosio::from_bin(row.data); + if (row.present) + table.put(obj); + else + table.erase(obj); + } +} + +template +void store_delta_kv(eosio::kv_environment environment, D& delta, F f) { + for (auto& row : delta.rows) { + f(); + auto obj = eosio::from_bin(row.data); + auto& obj0 = std::get(obj); +#warning uncomment this when we remove the database logic on rodeos + /* + if (row.present) + environment.kv_set(obj0.database.value, obj0.contract.value, obj0.key.pos, obj0.key.remaining(), + obj0.value.pos, obj0.value.remaining()); + else + environment.kv_erase(obj0.database.value, obj0.contract.value, obj0.key.pos, obj0.key.remaining()); + */ + } +} + +template +inline void store_delta(eosio::kv_environment environment, D& delta, bool bypass_preexist_check, F f) { + if (delta.name == "global_property") + store_delta_typed(environment, delta, bypass_preexist_check, f); + if (delta.name == "account") + store_delta_typed(environment, delta, bypass_preexist_check, f); + if (delta.name == "account_metadata") + store_delta_typed(environment, delta, bypass_preexist_check, f); + if (delta.name == "code") + store_delta_typed(environment, delta, bypass_preexist_check, f); + if (delta.name == "contract_table") + store_delta_typed(environment, delta, bypass_preexist_check, f); + if (delta.name == "contract_row") + store_delta_typed(environment, delta, bypass_preexist_check, f); + if (delta.name == "contract_index64") + store_delta_typed(environment, delta, bypass_preexist_check, f); + if (delta.name == "contract_index128") + store_delta_typed(environment, delta, bypass_preexist_check, f); + if (delta.name == "key_value") + store_delta_kv(environment, delta, f); +} + +inline void store_deltas(eosio::kv_environment environment, std::vector& deltas, + bool bypass_preexist_check) { + for (auto& delta : deltas) // + std::visit([&](auto& delta_any_v) { store_delta(environment, delta_any_v, bypass_preexist_check, [] {}); }, delta); +} + +} // namespace b1::rodeos diff --git a/libraries/rodeos/include/b1/rodeos/wasm_ql.hpp b/libraries/rodeos/include/b1/rodeos/wasm_ql.hpp new file mode 100644 index 00000000000..0791b0cc7c6 --- /dev/null +++ b/libraries/rodeos/include/b1/rodeos/wasm_ql.hpp @@ -0,0 +1,76 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace b1::rodeos::wasm_ql { + +class backend_cache; + +struct shared_state { + uint32_t max_console_size = {}; + uint32_t wasm_cache_size = {}; + uint64_t max_exec_time_ms = {}; + uint32_t max_action_return_value_size = {}; + std::string contract_dir = {}; + std::shared_ptr backend_cache = {}; + std::shared_ptr db; + + shared_state(std::shared_ptr db); + shared_state(const shared_state&) = delete; + ~shared_state(); + + shared_state& operator=(const shared_state&) = delete; +}; + +struct thread_state : action_state, console_state, query_state { + std::shared_ptr shared = {}; + eosio::vm::wasm_allocator wa = {}; +}; + +class thread_state_cache { + private: + std::mutex mutex; + std::shared_ptr shared_state; + std::vector> states; + + public: + thread_state_cache(const std::shared_ptr& shared_state) : shared_state(shared_state) {} + + std::unique_ptr get_state() { + std::lock_guard lock{ mutex }; + if (states.empty()) { + auto result = std::make_unique(); + result->shared = shared_state; + return result; + } + auto result = std::move(states.back()); + states.pop_back(); + return result; + } + + void store_state(std::unique_ptr state) { + std::lock_guard lock{ mutex }; + states.push_back(std::move(state)); + } +}; + +const std::vector& query_get_info(wasm_ql::thread_state& thread_state, + const std::vector& contract_kv_prefix); +const std::vector& query_get_block(wasm_ql::thread_state& thread_state, + const std::vector& contract_kv_prefix, std::string_view body); +const std::vector& query_get_abi(wasm_ql::thread_state& thread_state, const std::vector& contract_kv_prefix, + std::string_view body); +const std::vector& query_get_required_keys(wasm_ql::thread_state& thread_state, std::string_view body); +const std::vector& query_send_transaction(wasm_ql::thread_state& thread_state, + const std::vector& contract_kv_prefix, std::string_view body, + bool return_trace_on_except); +eosio::ship_protocol::transaction_trace_v0 +query_send_transaction(wasm_ql::thread_state& thread_state, const std::vector& contract_kv_prefix, + const eosio::ship_protocol::packed_transaction& trx, const rocksdb::Snapshot* snapshot, + std::vector>& memory, bool return_trace_on_except); + +} // namespace b1::rodeos::wasm_ql diff --git a/libraries/rodeos/include/eosio/datastream.hpp b/libraries/rodeos/include/eosio/datastream.hpp new file mode 100644 index 00000000000..317cbae2c8e --- /dev/null +++ b/libraries/rodeos/include/eosio/datastream.hpp @@ -0,0 +1,456 @@ +// TODO: this file duplicates a CDT file + +// clang-format off +/** + * @file datastream.hpp + * @copyright defined in eos/LICENSE + */ +#pragma once +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +//#include +//#include +//#include +//#include + +//#include +//#include + +namespace eosio { + +/** + * @defgroup datastream Data Stream + * @ingroup core + * @brief Defines data stream for reading and writing data in the form of bytes + */ + +/** + * A data stream for reading and writing data in the form of bytes + * + * @tparam T - Type of the datastream buffer + */ +template +class datastream { + public: + /** + * Construct a new datastream object + * + * @details Construct a new datastream object given the size of the buffer and start position of the buffer + * @param start - The start position of the buffer + * @param s - The size of the buffer + */ + datastream( T start, size_t s ) + :_start(start),_pos(start),_end(start+s){} + + bool check_available(size_t size) { + if (size > size_t(_end - _pos)) + return false; + return true; + } + + bool read_reuse_storage(const char*& result, size_t size) { + if (size > size_t(_end - _pos)) + return false; + result = _pos; + _pos += size; + return true; + } + + /** + * Skips a specified number of bytes from this stream + * + * @param s - The number of bytes to skip + */ + inline bool skip( size_t s ){ + _pos += s; + if (_pos > _end ) + return false; + return true; + } + + /** + * Reads a specified number of bytes from the stream into a buffer + * + * @param d - The pointer to the destination buffer + * @param s - the number of bytes to read + * @return true + */ + inline bool read( char* d, size_t s ) { + eosio::check( size_t(_end - _pos) >= (size_t)s, "read" ); + memcpy( d, _pos, s ); + _pos += s; + return true; + } + + /** + * Writes a specified number of bytes into the stream from a buffer + * + * @param d - The pointer to the source buffer + * @param s - The number of bytes to write + * @return true + */ + inline bool write( const char* d, size_t s ) { + eosio::check( _end - _pos >= (int32_t)s, "write" ); + memcpy( (void*)_pos, d, s ); + _pos += s; + return true; + } + + /** + * Writes a byte into the stream + * + * @brief Writes a byte into the stream + * @param c byte to write + * @return true + */ + inline bool put(char c) { + eosio::check( _pos < _end, "put" ); + *_pos = c; + ++_pos; + return true; + } + + inline bool write(char c) { return put(c); } + + /** + * Reads a byte from the stream + * + * @brief Reads a byte from the stream + * @param c - The reference to destination byte + * @return true + */ + inline bool get( unsigned char& c ) { return get( *(char*)&c ); } + + /** + * Reads a byte from the stream + * + * @brief Reads a byte from the stream + * @param c - The reference to destination byte + * @return true + */ + inline bool get( char& c ) + { + eosio::check( _pos < _end, "get" ); + c = *_pos; + ++_pos; + return true; + } + + /** + * Retrieves the current position of the stream + * + * @brief Retrieves the current position of the stream + * @return T - The current position of the stream + */ + T pos()const { return _pos; } + T get_pos()const { return _pos; } + inline bool valid()const { return _pos <= _end && _pos >= _start; } + + /** + * Sets the position within the current stream + * + * @brief Sets the position within the current stream + * @param p - The offset relative to the origin + * @return true if p is within the range + * @return false if p is not within the rawnge + */ + inline bool seekp(size_t p) { _pos = _start + p; return _pos <= _end; } + + /** + * Gets the position within the current stream + * + * @brief Gets the position within the current stream + * @return p - The position within the current stream + */ + inline size_t tellp()const { return size_t(_pos - _start); } + + /** + * Returns the number of remaining bytes that can be read/skipped + * + * @brief Returns the number of remaining bytes that can be read/skipped + * @return size_t - The number of remaining bytes + */ + inline size_t remaining()const { return _end - _pos; } + private: + /** + * The start position of the buffer + * + * @brief The start position of the buffer + */ + T _start; + /** + * The current position of the buffer + * + * @brief The current position of the buffer + */ + T _pos; + /** + * The end position of the buffer + * + * @brief The end position of the buffer + */ + T _end; +}; + +/** + * Specialization of datastream used to help determine the final size of a serialized value + */ +template<> +class datastream { + public: + /** + * Construct a new specialized datastream object given the initial size + * + * @param init_size - The initial size + */ + datastream( size_t init_size = 0):_size(init_size){} + + /** + * Increment the size by s. This behaves the same as write( const char* ,size_t s ). + * + * @param s - The amount of size to increase + * @return true + */ + inline bool skip( size_t s ) { _size += s; return true; } + + /** + * Increment the size by s. This behaves the same as skip( size_t s ) + * + * @param s - The amount of size to increase + * @return true + */ + inline bool write( const char* ,size_t s ) { _size += s; return true; } + + /** + * Increment the size by one + * + * @return true + */ + inline bool put(char ) { ++_size; return true; } + + inline bool write(char c) { return put(c); } + + /** + * Check validity. It's always valid + * + * @return true + */ + inline bool valid()const { return true; } + + /** + * Set new size + * + * @brief Set new size + * @param p - The new size + * @return true + */ + inline bool seekp(size_t p) { _size = p; return true; } + + /** + * Get the size + * + * @return size_t - The size + */ + inline size_t tellp()const { return _size; } + + /** + * Always returns 0 + * + * @return size_t - 0 + */ + inline size_t remaining()const { return 0; } + private: + /** + * The size used to determine the final size of a serialized value. + */ + size_t _size; +}; + +namespace _datastream_detail { + /** + * Check if type T is a pointer + * + * @brief Check if type T is a pointer + * @tparam T - The type to be checked + * @return true if T is a pointer + * @return false otherwise + */ + template + constexpr bool is_pointer() { + return std::is_pointer::value || + std::is_null_pointer::value || + std::is_member_pointer::value; + } + + /** + * Check if type T is a primitive type + * + * @brief Check if type T is a primitive type + * @tparam T - The type to be checked + * @return true if T is a primitive type + * @return false otherwise + */ + template + constexpr bool is_primitive() { + return std::is_arithmetic::value || + std::is_enum::value; + } + + /* + * Check if type T is a specialization of datastream + * + * @brief Check if type T is a datastream + * @tparam T - The type to be checked + */ + template + struct is_datastream { static constexpr bool value = false; }; + template + struct is_datastream> { static constexpr bool value = true; }; + + namespace operator_detection { + // These overloads will be ambiguous with the operator in eosio, unless + // the user has provided an operator that is a better match. + template::value>* = nullptr> + DataStream& operator>>( DataStream& ds, T& v ); + template::value>* = nullptr> + DataStream& operator<<( DataStream& ds, const T& v ); + template + using require_specialized_right_shift = std::void_t() >> std::declval())>; + template + using require_specialized_left_shift = std::void_t() << std::declval())>; + } + // Going through enable_if/is_detected is necessary for reasons that I don't entirely understand. + template + using require_specialized_right_shift = std::enable_if_t>; + template + using require_specialized_left_shift = std::enable_if_t>; +} + +/** + * Serialize a class + * + * @brief Serialize a class + * @param ds - The stream to write + * @param v - The value to serialize + * @tparam DataStream - Type of datastream + * @tparam T - Type of class + * @return DataStream& - Reference to the datastream + */ +template::value>* = nullptr> +DataStream& operator<<( DataStream& ds, const T& v ) { + to_bin(v, ds); + return ds; +} + +/* throws off gcc +// Backwards compatibility: allow user defined datastream operators to work with from_bin +template,T>* = nullptr> +void to_bin( const T& v, datastream& ds ) { + ds << v; +} +*/ + +/** + * Deserialize a class + * + * @brief Deserialize a class + * @param ds - The stream to read + * @param v - The destination for deserialized value + * @tparam DataStream - Type of datastream + * @tparam T - Type of class + * @return DataStream& - Reference to the datastream + */ +template::value>* = nullptr> +DataStream& operator>>( DataStream& ds, T& v ) { + from_bin(v, ds); + return ds; +} + +/* throws off gcc +template,T>* = nullptr> +void from_bin( T& v, datastream& ds ) { + ds >> v; +} +*/ + +/** + * Unpack data inside a fixed size buffer as T + * + * @ingroup datastream + * @brief Unpack data inside a fixed size buffer as T + * @tparam T - Type of the unpacked data + * @param buffer - Pointer to the buffer + * @param len - Length of the buffer + * @return T - The unpacked data + */ +template +T unpack( const char* buffer, size_t len ) { + T result; + datastream ds(buffer,len); + ds >> result; + return result; +} + +/** + * Unpack data inside a variable size buffer as T + * + * @ingroup datastream + * @brief Unpack data inside a variable size buffer as T + * @tparam T - Type of the unpacked data + * @param bytes - Buffer + * @return T - The unpacked data + */ +template +T unpack( const std::vector& bytes ) { + return unpack( bytes.data(), bytes.size() ); +} + +/** + * Get the size of the packed data + * + * @ingroup datastream + * @brief Get the size of the packed data + * @tparam T - Type of the data to be packed + * @param value - Data to be packed + * @return size_t - Size of the packed data + */ +template +size_t pack_size( const T& value ) { + datastream ps; + ps << value; + return ps.tellp(); +} + +/** + * Get packed data + * + * @ingroup datastream + * @brief Get packed data + * @tparam T - Type of the data to be packed + * @param value - Data to be packed + * @return bytes - The packed data + */ +template +std::vector pack( const T& value ) { + std::vector result; + result.resize(pack_size(value)); + + datastream ds( result.data(), result.size() ); + ds << value; + return result; +} +} diff --git a/libraries/rodeos/include/eosio/key_value.hpp b/libraries/rodeos/include/eosio/key_value.hpp new file mode 100644 index 00000000000..985d3adfb8a --- /dev/null +++ b/libraries/rodeos/include/eosio/key_value.hpp @@ -0,0 +1,958 @@ +// TODO: this file duplicates a CDT file + +// clang-format off +#pragma once +#include +#include +#include + +#include + +#include +#include +#include + +#define EOSIO_CDT_GET_RETURN_T(value_class, index_name) std::decay_t()))> + +/** + * @brief Macro to define an index. + * @details In the case where the autogenerated index names created by DEFINE_TABLE are not enough, a user can instead + * manually define the table and indices. This macro allows users to conveniently define an index without having to specify + * the index template type, as those can be large/unwieldy to type out. + * + * @param index_name - The index name. + * @param member_name - The name of the member pointer used for the index. This also defines the index's C++ variable name. + */ +#define KV_NAMED_INDEX(index_name, member_name) \ + index member_name{eosio::name{index_name}, &value_type::member_name}; + +namespace eosio { + namespace internal_use_do_not_use { + +#ifdef __eosio_cdt__ + +#define IMPORT extern "C" __attribute__((eosio_wasm_import)) + + // clang-format off + IMPORT void kv_erase(uint64_t db, uint64_t contract, const char* key, uint32_t key_size); + IMPORT void kv_set(uint64_t db, uint64_t contract, const char* key, uint32_t key_size, const char* value, uint32_t value_size); + IMPORT bool kv_get(uint64_t db, uint64_t contract, const char* key, uint32_t key_size, uint32_t& value_size); + IMPORT uint32_t kv_get_data(uint64_t db, uint32_t offset, char* data, uint32_t data_size); + IMPORT uint32_t kv_it_create(uint64_t db, uint64_t contract, const char* prefix, uint32_t size); + IMPORT void kv_it_destroy(uint32_t itr); + IMPORT int32_t kv_it_status(uint32_t itr); + IMPORT int32_t kv_it_compare(uint32_t itr_a, uint32_t itr_b); + IMPORT int32_t kv_it_key_compare(uint32_t itr, const char* key, uint32_t size); + IMPORT int32_t kv_it_move_to_end(uint32_t itr); + IMPORT int32_t kv_it_next(uint32_t itr); + IMPORT int32_t kv_it_prev(uint32_t itr); + IMPORT int32_t kv_it_lower_bound(uint32_t itr, const char* key, uint32_t size); + IMPORT int32_t kv_it_key(uint32_t itr, uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size); + IMPORT int32_t kv_it_value(uint32_t itr, uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size); + // xclang-format on + +#undef IMPORT + +#endif + } + +#ifdef __eosio_cdt__ +class kv_environment { + public: + kv_environment() {} + + void kv_set(const std::vector& k, const std::vector& v) { + internal_use_do_not_use::kv_set(k.data(), k.size(), v.data(), v.size()); + } + + // clang-format off + void kv_erase(uint64_t db, uint64_t contract, const char* key, uint32_t key_size) {return internal_use_do_not_use::kv_erase(db, contract, key, key_size);} + void kv_set(uint64_t db, uint64_t contract, const char* key, uint32_t key_size, const char* value, uint32_t value_size) {return internal_use_do_not_use::kv_set(db, contract, key, key_size, value, value_size);} + bool kv_get(uint64_t db, uint64_t contract, const char* key, uint32_t key_size, uint32_t& value_size) {return internal_use_do_not_use::kv_get(db, contract, key, key_size, value_size);} + uint32_t kv_get_data(uint64_t db, uint32_t offset, char* data, uint32_t data_size) {return internal_use_do_not_use::kv_get_data(db, offset, data, data_size);} + uint32_t kv_it_create(uint64_t db, uint64_t contract, const char* prefix, uint32_t size) {return internal_use_do_not_use::kv_it_create(db, contract, prefix, size);} + void kv_it_destroy(uint32_t itr) {return internal_use_do_not_use::kv_it_destroy(itr);} + int32_t kv_it_status(uint32_t itr) {return internal_use_do_not_use::kv_it_status(itr);} + int32_t kv_it_compare(uint32_t itr_a, uint32_t itr_b) {return internal_use_do_not_use::kv_it_compare(itr_a, itr_b);} + int32_t kv_it_key_compare(uint32_t itr, const char* key, uint32_t size) {return internal_use_do_not_use::kv_it_key_compare(itr, key, size);} + int32_t kv_it_move_to_end(uint32_t itr) {return internal_use_do_not_use::kv_it_move_to_end(itr);} + int32_t kv_it_next(uint32_t itr) {return internal_use_do_not_use::kv_it_next(itr);} + int32_t kv_it_prev(uint32_t itr) {return internal_use_do_not_use::kv_it_prev(itr);} + int32_t kv_it_lower_bound(uint32_t itr, const char* key, uint32_t size) {return internal_use_do_not_use::kv_it_lower_bound(itr, key, size);} + int32_t kv_it_key(uint32_t itr, uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) {return internal_use_do_not_use::kv_it_key(itr, offset, dest, size, actual_size);} + int32_t kv_it_value(uint32_t itr, uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) {return internal_use_do_not_use::kv_it_value(itr, offset, dest, size, actual_size);} + // xclang-format on +}; +#endif + +namespace detail { + constexpr inline size_t max_stack_buffer_size = 512; + + template + static void serialize(const V& value, void* buffer, size_t size) { + datastream ds((char*)buffer, size); + unsigned_int i{0}; + ds << i; + ds << value; + } + + template + static void serialize(const std::variant& value, void* buffer, size_t size) { + datastream ds((char*)buffer, size); + ds << value; + } + + template + static void deserialize(V& value, void* buffer, size_t size) { + unsigned_int idx; + datastream ds((char*)buffer, size); + + ds >> idx; + eosio::check(idx==unsigned_int(0), "there was an error deserializing this value."); + ds >> value; + } + + template + static void deserialize(std::variant& value, void* buffer, size_t size) { + datastream ds((char*)buffer, size); + ds >> value; + } + + template + static size_t get_size(const V& value) { + auto size = pack_size(value); + return size + 1; + } + + template + static size_t get_size(const std::variant& value) { + auto size = pack_size(value); + return size; + } +} + +/** + * The key_type struct is used to store the binary representation of a key. + */ +struct key_type : private std::vector { + key_type() = default; + + explicit key_type(std::vector&& v) : std::vector(v) {} + + key_type(char* str, size_t size) : std::vector(str, str+size) {} + + key_type operator+(const key_type& b) const { + key_type ret = *this; + ret += b; + return ret; + } + + key_type& operator+=(const key_type& b) { + this->insert(this->end(), b.begin(), b.end()); + return *this; + } + + static key_type from_hex( const std::string_view& str ) { + key_type out; + + check( str.size() % 2 == 0, "invalid hex string length" ); + out.reserve( str.size() / 2 ); + + auto start = str.data(); + auto end = start + str.size(); + for(const char* p = start; p != end; p+=2 ) { + auto hic = p[0]; + auto lowc = p[1]; + + uint8_t hi = hic <= '9' ? hic-'0' : 10+(hic-'a'); + uint8_t low = lowc <= '9' ? lowc-'0' : 10+(lowc-'a'); + + out.push_back( char((hi << 4) | low) ); + } + + return out; + } + + std::string to_hex() const { + const char* hex_characters = "0123456789abcdef"; + + uint32_t buffer_size = 2 * size(); + check(buffer_size >= size(), "length passed into printhex is too large"); + + void* buffer = buffer_size > detail::max_stack_buffer_size ? malloc(buffer_size) : alloca(buffer_size); + + char* b = reinterpret_cast(buffer); + const uint8_t* d = reinterpret_cast(data()); + for(uint32_t i = 0; i < size(); ++i) { + *b = hex_characters[d[i] >> 4]; + ++b; + *b = hex_characters[d[i] & 0x0f]; + ++b; + } + + std::string ret{reinterpret_cast(buffer), buffer_size}; + + if (buffer_size > detail::max_stack_buffer_size) { + free(buffer); + } + + return ret; + } + + using std::vector::data; + using std::vector::size; + using std::vector::resize; +}; + +/* @cond PRIVATE */ +template +inline key_type make_key(T&& t) { + return key_type(convert_to_key(std::forward(t))); +} + +inline key_type make_prefix(eosio::name table_name, eosio::name index_name, uint8_t status = 1) { + return make_key(std::make_tuple(status, table_name, index_name)); +} + +inline key_type table_key(const key_type& prefix, const key_type& key) { + return prefix + key; +} +/* @endcond */ + +// This is the "best" way to document a function that does not technically exist using Doxygen. +#if EOSIO_CDT_DOXYGEN +/** + * @brief A function for converting types to the appropriate binary representation for the EOSIO Key Value database. + * @details The CDT provides implementations of this function for many of the common primitives and for structs/tuples. + * If sticking with standard types, contract developers should not need to interact with this function. + * If doing something more advanced, contract developers may need to provide their own implementation for a special type. + */ +template +inline key_type make_key(T val) { + return {}; +} +#endif + +static constexpr eosio::name kv_ram = "eosio.kvram"_n; +static constexpr eosio::name kv_disk = "eosio.kvdisk"_n; + +struct default_constructor_tag; + +/** + * @defgroup keyvalue Key Value Table + * @ingroup contracts + * + * @brief Defines an EOSIO Key Value Table + * @details EOSIO Key Value API provides a C++ interface to the EOSIO Key Value database. + * Key Value Tables require 1 primary index, of any type that can be serialized to a binary representation. + * Key Value Tables support 0 or more secondary index, of any type that can be serialized to a binary representation. + * Indexes must be a member variable or a member function. + * + * @tparam T - the type of the data stored as the value of the table + */ +template +class kv_table { + + class kv_index; + + class iterator { + public: + enum class status { + iterator_ok = 0, // Iterator is positioned at a key-value pair + iterator_erased = -1, // The key-value pair that the iterator used to be positioned at was erased + iterator_end = -2, // Iterator is out-of-bounds + }; + + iterator() = default; + + iterator(uint32_t itr, status itr_stat, const kv_index* index) : itr{itr}, itr_stat{itr_stat}, index{index} {} + + iterator(iterator&& other) : + itr(std::exchange(other.itr, 0)), + itr_stat(std::move(other.itr_stat)) + {} + + ~iterator() { + if (itr) { + index->tbl->environment.kv_it_destroy(itr); + } + } + + iterator& operator=(iterator&& other) { + if (itr) { + index->tbl->environment.kv_it_destroy(itr); + } + itr = std::exchange(other.itr, 0); + itr_stat = std::move(other.itr_stat); + return *this; + } + + bool good()const { return itr_stat != status::iterator_end; } + + /** + * Returns the value that the iterator points to. + * @ingroup keyvalue + * + * @return The value that the iterator points to. + */ + T value() const { + using namespace detail; + + eosio::check(itr_stat != status::iterator_end, "Cannot read end iterator"); + + uint32_t value_size; + uint32_t actual_value_size; + uint32_t actual_data_size; + uint32_t offset = 0; + + // call once to get the value_size + index->tbl->environment.kv_it_value(itr, 0, (char*)nullptr, 0, value_size); + + void* buffer = value_size > detail::max_stack_buffer_size ? malloc(value_size) : alloca(value_size); + auto stat = index->tbl->environment.kv_it_value(itr, offset, (char*)buffer, value_size, actual_value_size); + + eosio::check(static_cast(stat) == status::iterator_ok, "Error reading value"); + + void* deserialize_buffer = buffer; + size_t deserialize_size = actual_value_size; + + bool is_primary = index->index_name == index->tbl->primary_index_name; + if (!is_primary) { + auto success = index->tbl->environment.kv_get(index->tbl->db_name, index->contract_name.value, (char*)buffer, actual_value_size, actual_data_size); + eosio::check(success, "failure getting primary key in `value()`"); + + void* pk_buffer = actual_data_size > detail::max_stack_buffer_size ? malloc(actual_data_size) : alloca(actual_data_size); + index->tbl->environment.kv_get_data(index->tbl->db_name, 0, (char*)pk_buffer, actual_data_size); + + deserialize_buffer = pk_buffer; + deserialize_size = actual_data_size; + } + + T val; + detail::deserialize(val, deserialize_buffer, deserialize_size); + + if (value_size > detail::max_stack_buffer_size) { + free(buffer); + } + + if (!is_primary && actual_data_size > detail::max_stack_buffer_size) { + free(deserialize_buffer); + } + return val; + } + + key_type key() const { + uint32_t actual_value_size; + uint32_t value_size; + + // call once to get the value size + index->tbl->environment.kv_it_key(itr, 0, (char*)nullptr, 0, value_size); + + void* buffer = value_size > detail::max_stack_buffer_size ? malloc(value_size) : alloca(value_size); + auto stat = index->tbl->environment.kv_it_key(itr, 0, (char*)buffer, value_size, actual_value_size); + + eosio::check(static_cast(stat) == status::iterator_ok, "Error getting key"); + + return {(char*)buffer, actual_value_size}; + } + + iterator& operator++() { + eosio::check(itr_stat != status::iterator_end, "cannot increment end iterator"); + itr_stat = static_cast(index->tbl->environment.kv_it_next(itr)); + return *this; + } + + iterator& operator--() { + if (!itr) { + itr = index->tbl->environment.kv_it_create(index->tbl->db_name, index->contract_name.value, index->prefix.data(), index->prefix.size()); + } + itr_stat = static_cast(index->tbl->environment.kv_it_prev(itr)); + eosio::check(itr_stat != status::iterator_end, "decremented past the beginning"); + return *this; + } + + int32_t key_compare(key_type kt) const { + if (itr == 0 || itr_stat == status::iterator_end) { + return 1; + } else { + return index->tbl->environment.kv_it_key_compare(itr, kt.data(), kt.size()); + } + } + + bool operator==(const iterator& b) const { + return compare(b) == 0; + } + + bool operator!=(const iterator& b) const { + return compare(b) != 0; + } + + bool operator<(const iterator& b) const { + return compare(b) < 0; + } + + bool operator<=(const iterator& b) const { + return compare(b) <= 0; + } + + bool operator>(const iterator& b) const { + return compare(b) > 0; + } + + bool operator>=(const iterator& b) const { + return compare(b) >= 0; + } + + private: + uint32_t itr; + status itr_stat; + + const kv_index* index; + + int compare(const iterator& b) const { + bool a_is_end = !itr || itr_stat == status::iterator_end; + bool b_is_end = !b.itr || b.itr_stat == status::iterator_end; + if (a_is_end && b_is_end) { + return 0; + } else if (a_is_end && b.itr) { + return 1; + } else if (itr && b_is_end) { + return -1; + } else { + return index->tbl->environment.kv_it_compare(itr, b.itr); + } + } + }; + + class kv_index { + + public: + eosio::name index_name; + eosio::name table_name; + eosio::name contract_name; + + key_type to_table_key( const key_type& k )const{ return table_key( prefix, k ); } + + protected: + kv_index() = default; + + template + kv_index(eosio::name index_name, KF&& kf) : index_name{index_name} { + key_function = [=](const T& t) { + return make_key(std::invoke(kf, &t)); + }; + } + + key_type get_key(const T& inst) const { return key_function(inst); } + kv_table* tbl; + key_type prefix; + + private: + friend kv_table; + + std::function key_function; + + virtual void setup() = 0; + }; + +public: + + using iterator = kv_table::iterator; + using value_type = T; + + /** + * @ingroup keyvalue + * + * @brief Defines an index on an EOSIO Key Value Table + * @details A Key Value Index allows a user of the table to search based on a given field. + * The only restrictions on that field are that it is serializable to a binary representation sortable by the KV intrinsics. + * Convenience functions exist to handle most of the primitive types as well as some more complex types, and are + * used automatically where possible. + * + * @tparam K - The type of the key used in the index. + */ + template + class index : public kv_index { + public: + using kv_table::kv_index::tbl; + using kv_table::kv_index::table_name; + using kv_table::kv_index::contract_name; + using kv_table::kv_index::index_name; + using kv_table::kv_index::prefix; + + template + index(eosio::name name, KF&& kf) : kv_index{name, kf} { + static_assert(std::is_same_v()))>>>, + "Make sure the variable/function passed to the constructor returns the same type as the template parameter."); + } + + /** + * Search for an existing object in a table by the index, using the given key. + * @ingroup keyvalue + * + * @param key - The key to search for. + * @return An iterator to the found object OR the `end` iterator if the given key was not found. + */ + iterator find(const K& key) const { + auto t_key = table_key(prefix, make_key(key)); + + return find(t_key); + } + + iterator find(const key_type& key) const { + uint32_t itr = tbl->environment.kv_it_create(tbl->db_name, contract_name.value, prefix.data(), prefix.size()); + int32_t itr_stat = tbl->environment.kv_it_lower_bound(itr, key.data(), key.size()); + + auto cmp = tbl->environment.kv_it_key_compare(itr, key.data(), key.size()); + + if (cmp != 0) { + tbl->environment.kv_it_destroy(itr); + return end(); + } + + return {itr, static_cast(itr_stat), this}; + } + + /** + * Check if a given key exists in the index. + * @ingroup keyvalue + * + * @param key - The key to check for. + * @return If the key exists or not. + */ + bool exists(const K& key) const { + auto t_key = table_key(prefix, make_key(key)); + return exists(t_key); + } + + bool exists(const key_type& key) const { + uint32_t value_size; + return tbl->environment.kv_get(tbl->db_name, contract_name.value, key.data(), key.size(), value_size); + } + + /** + * Get the value for an existing object in a table by the index, using the given key. + * @ingroup keyvalue + * + * @param key - The key to search for. + * @return The value corresponding to the key. + */ + T operator[](const K& key) const { + return operator[](make_key(key)); + } + + T operator[](const key_type& key) const { + auto opt = get(key); + eosio::check(opt, __FILE__ ":" + std::to_string(__LINE__) + " Key not found in `[]`"); + return *opt; + } + + /** + * Get the value for an existing object in a table by the index, using the given key. + * @ingroup keyvalue + * + * @param key - The key to search for. + * @return A std::optional of the value corresponding to the key. + */ + std::optional get(const K& key) const { + return get(make_key(key)); + } + + std::optional get(const key_type& k ) const { + auto key = table_key( prefix, k ); + uint32_t value_size; + uint32_t actual_data_size; + std::optional ret_val; + + auto success = tbl->environment.kv_get(tbl->db_name, contract_name.value, key.data(), key.size(), value_size); + if (!success) { + return ret_val; + } + + void* buffer = value_size > detail::max_stack_buffer_size ? malloc(value_size) : alloca(value_size); + auto copy_size = tbl->environment.kv_get_data(tbl->db_name, 0, (char*)buffer, value_size); + + void* deserialize_buffer = buffer; + size_t deserialize_size = copy_size; + + bool is_primary = index_name == tbl->primary_index_name; + if (!is_primary) { + auto success = tbl->environment.kv_get(tbl->db_name, contract_name.value, (char*)buffer, copy_size, actual_data_size); + eosio::check(success, "failure getting primary key"); + + void* pk_buffer = actual_data_size > detail::max_stack_buffer_size ? malloc(actual_data_size) : alloca(actual_data_size); + auto pk_copy_size = tbl->environment.kv_get_data(tbl->db_name, 0, (char*)pk_buffer, actual_data_size); + + deserialize_buffer = pk_buffer; + deserialize_size = pk_copy_size; + } + + ret_val.emplace(); + detail::deserialize(*ret_val, deserialize_buffer, deserialize_size); + + if (value_size > detail::max_stack_buffer_size) { + free(buffer); + } + + if (is_primary && actual_data_size > detail::max_stack_buffer_size) { + free(deserialize_buffer); + } + + return ret_val; + } + + /** + * Returns an iterator to the object with the lowest key (by this index) in the table. + * @ingroup keyvalue + * + * @return An iterator to the object with the lowest key (by this index) in the table. + */ + iterator begin() const { + uint32_t itr = tbl->environment.kv_it_create(tbl->db_name, contract_name.value, prefix.data(), prefix.size()); + int32_t itr_stat = tbl->environment.kv_it_lower_bound(itr, "", 0); + + return {itr, static_cast(itr_stat), this}; + } + + /** + * Returns an iterator pointing past the end. It does not point to any element, therefore `value` should not be called on it. + * @ingroup keyvalue + * + * @return An iterator pointing past the end. + */ + iterator end() const { + return {0, iterator::status::iterator_end, this}; + } + + /** + * Returns an iterator pointing to the element with the lowest key greater than or equal to the given key. + * @ingroup keyvalue + * + * @return An iterator pointing to the element with the lowest key greater than or equal to the given key. + */ + iterator lower_bound(const K& key) const { + return lower_bound(make_key(key)); + } + + iterator lower_bound(const key_type& k ) const { + auto key = table_key( prefix, k ); + uint32_t itr = tbl->environment.kv_it_create(tbl->db_name, contract_name.value, prefix.data(), prefix.size()); + int32_t itr_stat = tbl->environment.kv_it_lower_bound(itr, key.data(), key.size()); + + return {itr, static_cast(itr_stat), this}; + } + + /** + * Returns an iterator pointing to the first element greater than the given key. + * @ingroup keyvalue + * + * @return An iterator pointing to the first element greater than the given key. + */ + iterator upper_bound(const K& key) const { + return upper_bound(make_key(key)); + } + + iterator upper_bound(const key_type& key) const { + auto it = lower_bound(key); + + auto cmp = it.key_compare(key); + if (cmp == 0) { + ++it; + } + + return it; + } + + /** + * Returns a vector of objects that fall between the specifed range. The range is inclusive, exclusive. + * @ingroup keyvalue + * + * @param begin - The beginning of the range (inclusive). + * @param end - The end of the range (exclusive). + * @return A vector containing all the objects that fall between the range. + */ + std::vector range(const K& b, const K& e) const { + return range(make_key(b), make_key(e)); + } + + std::vector range(const key_type& b_key, const key_type& e_key) const { + auto b = table_key(prefix, make_key(b_key)); + auto e = table_key(prefix, make_key(e_key)); + std::vector return_values; + + for(auto itr = lower_bound(b), end_itr = lower_bound(e); itr < end_itr; ++itr) { + return_values.push_back(itr.value()); + } + + return return_values; + } + + void setup() override { + prefix = make_prefix(table_name, index_name); + } + }; + + /** + * @ingroup keyvalue + * Puts a value into the table. If the value already exists, it updates the existing entry. + * The key is determined from the defined primary index. + * If the put attempts to store over an existing secondary index, the transaction will be aborted. + * + * @param value - The entry to be stored in the table. + */ + void put(const T& value) { + uint32_t value_size; + T old_value; + + auto primary_key = primary_index->get_key(value); + auto tbl_key = table_key(make_prefix(table_name, primary_index->index_name), primary_key); + + auto primary_key_found = environment.kv_get(db_name, contract_name.value, tbl_key.data(), tbl_key.size(), value_size); + + if (primary_key_found) { + void* buffer = value_size > detail::max_stack_buffer_size ? malloc(value_size) : alloca(value_size); + auto copy_size = environment.kv_get_data(db_name, 0, (char*)buffer, value_size); + + detail::deserialize(old_value, buffer, copy_size); + + if (value_size > detail::max_stack_buffer_size) { + free(buffer); + } + } + + for (const auto& idx : secondary_indices) { + auto sec_tbl_key = table_key(make_prefix(table_name, idx->index_name), idx->get_key(value)); + auto sec_found = environment.kv_get(db_name, contract_name.value, sec_tbl_key.data(), sec_tbl_key.size(), value_size); + + if (!primary_key_found) { + eosio::check(!sec_found, "Attempted to store an existing secondary index."); + environment.kv_set(db_name, contract_name.value, sec_tbl_key.data(), sec_tbl_key.size(), tbl_key.data(), tbl_key.size()); + } else { + if (sec_found) { + void* buffer = value_size > detail::max_stack_buffer_size ? malloc(value_size) : alloca(value_size); + auto copy_size = environment.kv_get_data(db_name, 0, (char*)buffer, value_size); + + auto res = memcmp(buffer, tbl_key.data(), copy_size); + eosio::check(copy_size == tbl_key.size() && res == 0, "Attempted to update an existing secondary index."); + + if (copy_size > detail::max_stack_buffer_size) { + free(buffer); + } + } else { + auto old_sec_key = table_key(make_prefix(table_name, idx->index_name), idx->get_key(old_value)); + environment.kv_erase(db_name, contract_name.value, old_sec_key.data(), old_sec_key.size()); + environment.kv_set(db_name, contract_name.value, sec_tbl_key.data(), sec_tbl_key.size(), tbl_key.data(), tbl_key.size()); + } + } + + } + + size_t data_size = detail::get_size(value); + void* data_buffer = data_size > detail::max_stack_buffer_size ? malloc(data_size) : alloca(data_size); + + detail::serialize(value, data_buffer, data_size); + + environment.kv_set(db_name, contract_name.value, tbl_key.data(), tbl_key.size(), (const char*)data_buffer, data_size); + + if (data_size > detail::max_stack_buffer_size) { + free(data_buffer); + } + } + + /** + * Removes a value from the table. + * @ingroup keyvalue + * + * @param key - The key of the value to be removed. + */ + void erase(const T& value) { + uint32_t value_size; + + auto primary_key = primary_index->get_key(value); + auto tbl_key = table_key(make_prefix(table_name, primary_index->index_name), primary_key); + auto primary_key_found = environment.kv_get(db_name, contract_name.value, tbl_key.data(), tbl_key.size(), value_size); + + if (!primary_key_found) { + return; + } + + for (const auto& idx : secondary_indices) { + auto sec_tbl_key = table_key(make_prefix(table_name, idx->index_name), idx->get_key(value)); + environment.kv_erase(db_name, contract_name.value, sec_tbl_key.data(), sec_tbl_key.size()); + } + + environment.kv_erase(db_name, contract_name.value, tbl_key.data(), tbl_key.size()); + } + +protected: + kv_table() = default; + + kv_environment environment; + + kv_table(kv_environment environment) : environment{std::move(environment)} {} + + kv_table(const kv_table&) = delete; + kv_table(kv_table&&) = delete; + + template + void setup_indices(I& index) { + kv_index* idx = &index; + idx->contract_name = contract_name; + idx->table_name = table_name; + idx->tbl = this; + + idx->setup(); + secondary_indices.push_back(idx); + } + + template + void init(eosio::name contract, eosio::name table, eosio::name db, PrimaryIndex& prim_index, SecondaryIndices&... indices) { + validate_types(prim_index); + (validate_types(indices), ...); + + contract_name = contract; + table_name = table; + db_name = db.value; + + primary_index = &prim_index; + primary_index->contract_name = contract_name; + primary_index->table_name = table_name; + primary_index->tbl = this; + + primary_index->setup(); + + primary_index_name = primary_index->index_name; + + (setup_indices(indices), ...); + } + +private: + eosio::name contract_name; + eosio::name table_name; + uint64_t db_name; + + eosio::name primary_index_name; + + kv_index* primary_index; + std::vector secondary_indices; + + constexpr void validate_types() {} + + template + constexpr void validate_types(Type& t) { + constexpr bool is_kv_index = std::is_base_of_v>; + static_assert(is_kv_index, "Incorrect type passed to init. Must be a reference to an index."); + } + +}; + +template +class kv_singleton { + kv_environment environment; + + struct state { + T value; + char* raw_original; + size_t raw_original_size; + + bool is_dirty = false; + bool is_cached = false; + + ~state() { + if (raw_original_size) { + free(raw_original); + } + } + }; + +public: + using value_type = T; + + explicit kv_singleton(eosio::name contract_name, bool write_on_destruct) : contract_name{contract_name}, write_on_destruct{write_on_destruct} { + key = make_prefix(); + } + + kv_singleton(eosio::name contract_name, kv_environment environment, bool write_on_destruct) : environment{environment}, contract_name{contract_name}, write_on_destruct{write_on_destruct} {} + + ~kv_singleton() { + if (get_state().is_dirty && write_on_destruct) { + store(); + } + } + + const T& get() { + auto& ste = get_state(); + if (!ste.is_cached) { + uint32_t copy_size; + uint32_t value_size; + + auto success = environment.kv_get(db_name, contract_name.value, key.data(), key.size(), value_size); + + eosio::check(success, "tried to get a singleton that does not exist"); + + ste.raw_original = (char*)malloc(value_size); + ste.raw_original_size = value_size; + copy_size = environment.kv_get_data(db_name, 0, ste.raw_original, value_size); + + detail::deserialize(ste.value, ste.raw_original, copy_size); + ste.is_cached = true; + } + + return get_state().value; + } + + void set(const T& val) { + auto& ste = get_state(); + ste.value = val; + ste.is_dirty = true; + ste.is_cached = true; + } + + bool exists() { + uint32_t value_size; + + return environment.kv_get(db_name, contract_name.value, key.data(), key.size(), value_size); + } + + void erase() { + environment.kv_erase(db_name, contract_name.value, key.data(), key.size()); + auto& ste = get_state(); + ste.is_cached = false; + ste.is_dirty = false; + ste.raw_original_size = 0; + free(ste.raw_original); + } + + void store() { + auto& ste = get_state(); + if (ste.is_dirty) { + size_t data_size = detail::get_size(ste.value); + void* data_buffer = data_size > detail::max_stack_buffer_size ? malloc(data_size) : alloca(data_size); + + detail::serialize(ste.value, data_buffer, data_size); + + if (ste.raw_original_size != data_size || memcmp(ste.raw_original, data_buffer, data_size) != 0) { + environment.kv_set(db_name, contract_name.value, key.data(), key.size(), (const char*)data_buffer, data_size); + } + } + } + +private: + constexpr static uint64_t db_name = static_cast(DbName); + constexpr static uint64_t singleton_name = static_cast(SingletonName); + + eosio::name contract_name; + bool write_on_destruct; + key_type key; + + key_type make_prefix() { + return make_key(std::make_tuple(0x02, singleton_name)); + } + + state& get_state() { + static state value; + return value; + } +}; + +} // eosio diff --git a/libraries/rodeos/rodeos.cpp b/libraries/rodeos/rodeos.cpp new file mode 100644 index 00000000000..48cb5c62329 --- /dev/null +++ b/libraries/rodeos/rodeos.cpp @@ -0,0 +1,329 @@ +#include + +#include +#include +#include + +namespace b1::rodeos { + +namespace ship_protocol = eosio::ship_protocol; + +using ship_protocol::get_blocks_result_base; +using ship_protocol::get_blocks_result_v0; +using ship_protocol::get_blocks_result_v1; +using ship_protocol::signed_block_header; +using ship_protocol::signed_block_variant; + +rodeos_db_snapshot::rodeos_db_snapshot(std::shared_ptr partition, bool persistent) + : partition{ std::move(partition) }, db{ this->partition->db } { + if (persistent) { + undo_stack.emplace(*db, this->partition->undo_prefix); + write_session.emplace(*db); + } else { + snap.emplace(db->rdb.get()); + write_session.emplace(*db, snap->snapshot()); + } + + db_view_state view_state{ state_account, *db, *write_session, this->partition->contract_kv_prefix }; + fill_status_sing sing{ state_account, view_state, false }; + if (sing.exists()) { + auto status = std::get<0>(sing.get()); + chain_id = status.chain_id; + head = status.head; + head_id = status.head_id; + irreversible = status.irreversible; + irreversible_id = status.irreversible_id; + first = status.first; + } +} + +void rodeos_db_snapshot::refresh() { + if (undo_stack) + throw std::runtime_error("can not refresh a persistent snapshot"); + snap.emplace(db->rdb.get()); + write_session->snapshot = snap->snapshot(); + write_session->wipe_cache(); +} + +void rodeos_db_snapshot::write_fill_status() { + if (!undo_stack) + throw std::runtime_error("Can only write to persistent snapshots"); + fill_status status; + if (irreversible < head) + status = fill_status_v0{ .chain_id = chain_id, + .head = head, + .head_id = head_id, + .irreversible = irreversible, + .irreversible_id = irreversible_id, + .first = first }; + else + status = fill_status_v0{ .chain_id = chain_id, + .head = head, + .head_id = head_id, + .irreversible = head, + .irreversible_id = head_id, + .first = first }; + + db_view_state view_state{ state_account, *db, *write_session, partition->contract_kv_prefix }; + view_state.kv_state.enable_write = true; + fill_status_sing sing{ state_account, view_state, false }; + sing.set(status); + sing.store(); +} + +void rodeos_db_snapshot::end_write(bool write_fill) { + if (!undo_stack) + throw std::runtime_error("Can only write to persistent snapshots"); + if (write_fill) + write_fill_status(); + write_session->write_changes(*undo_stack); +} + +void rodeos_db_snapshot::start_block(const get_blocks_result_base& result) { + if (!undo_stack) + throw std::runtime_error("Can only write to persistent snapshots"); + if (!result.this_block) + throw std::runtime_error("get_blocks_result this_block is empty"); + + if (result.this_block->block_num <= head) { + ilog("switch forks at block ${b}; database contains revisions ${f} - ${h}", + ("b", result.this_block->block_num)("f", undo_stack->first_revision())("h", undo_stack->revision())); + if (undo_stack->first_revision() >= result.this_block->block_num) + throw std::runtime_error("can't switch forks since database doesn't contain revision " + + std::to_string(result.this_block->block_num - 1)); + write_session->wipe_cache(); + while (undo_stack->revision() >= result.this_block->block_num) // + undo_stack->undo(true); + } + + if (head_id != eosio::checksum256{} && (!result.prev_block || result.prev_block->block_id != head_id)) + throw std::runtime_error("prev_block does not match"); + + if (result.this_block->block_num <= result.last_irreversible.block_num) { + undo_stack->commit(std::min(result.last_irreversible.block_num, head)); + undo_stack->set_revision(result.this_block->block_num, false); + } else { + end_write(false); + undo_stack->commit(std::min(result.last_irreversible.block_num, head)); + undo_stack->push(false); + } + writing_block = result.this_block->block_num; +} + +void rodeos_db_snapshot::end_block(const get_blocks_result_base& result, bool force_write) { + if (!undo_stack) + throw std::runtime_error("Can only write to persistent snapshots"); + if (!result.this_block) + throw std::runtime_error("get_blocks_result this_block is empty"); + if (!writing_block || result.this_block->block_num != *writing_block) + throw std::runtime_error("call start_block first"); + + bool near = result.this_block->block_num + 4 >= result.last_irreversible.block_num; + bool write_now = !(result.this_block->block_num % 200) || near || force_write; + head = result.this_block->block_num; + head_id = result.this_block->block_id; + irreversible = result.last_irreversible.block_num; + irreversible_id = result.last_irreversible.block_id; + if (!first || head < first) + first = head; + if (write_now) + end_write(write_now); + if (near) + db->flush(false, false); +} + +void rodeos_db_snapshot::check_write(const ship_protocol::get_blocks_result_base& result) { + if (!undo_stack) + throw std::runtime_error("Can only write to persistent snapshots"); + if (!result.this_block) + throw std::runtime_error("get_blocks_result this_block is empty"); + if (!writing_block || result.this_block->block_num != *writing_block) + throw std::runtime_error("call start_block first"); +} + +void rodeos_db_snapshot::write_block_info(uint32_t block_num, const eosio::checksum256& id, + const eosio::ship_protocol::signed_block_header& block) { + db_view_state view_state{ state_account, *db, *write_session, partition->contract_kv_prefix }; + view_state.kv_state.enable_write = true; + + block_info_v0 info; + info.num = block_num; + info.id = id; + info.timestamp = block.timestamp; + info.producer = block.producer; + info.confirmed = block.confirmed; + info.previous = block.previous; + info.transaction_mroot = block.transaction_mroot; + info.action_mroot = block.action_mroot; + info.schedule_version = block.schedule_version; + info.new_producers = block.new_producers; + info.producer_signature = block.producer_signature; + + block_info_kv table{ kv_environment{ view_state } }; + table.put(info); +} + +namespace { + std::string to_string( const eosio::checksum256& cs ) { + auto bytes = cs.extract_as_byte_array(); + return fc::to_hex((const char*)bytes.data(), bytes.size()); + } +} + +void rodeos_db_snapshot::write_block_info(const ship_protocol::get_blocks_result_v0& result) { + check_write(result); + if (!result.block) + return; + + uint32_t block_num = result.this_block->block_num; + eosio::input_stream bin = *result.block; + signed_block_header block; + from_bin(block, bin); + + auto blk_trace = fc_create_trace_with_id( "Block", result.this_block->block_id ); + auto blk_span = fc_create_span( blk_trace, "rodeos-received" ); + fc_add_tag( blk_span, "block_id", to_string( result.this_block->block_id ) ); + fc_add_tag( blk_span, "block_num", block_num ); + fc_add_tag( blk_span, "block_time", block.timestamp.to_time_point().elapsed.count() ); + + write_block_info(block_num, result.this_block->block_id, block); +} + +void rodeos_db_snapshot::write_block_info(const ship_protocol::get_blocks_result_v1& result) { + check_write(result); + if (!result.block) + return; + + uint32_t block_num = result.this_block->block_num; + + const signed_block_header& header = + std::visit([](const auto& blk) { return static_cast(blk); }, *result.block); + + auto blk_trace = fc_create_trace_with_id( "Block", result.this_block->block_id ); + auto blk_span = fc_create_span( blk_trace, "rodeos-received" ); + fc_add_tag( blk_span, "block_id", to_string( result.this_block->block_id ) ); + fc_add_tag( blk_span, "block_num", block_num ); + fc_add_tag( blk_span, "block_time", eosio::microseconds_to_str( header.timestamp.to_time_point().elapsed.count() ) ); + + write_block_info(block_num, result.this_block->block_id, header); +} + +void rodeos_db_snapshot::write_deltas(uint32_t block_num, eosio::opaque> deltas, std::function shutdown) { + db_view_state view_state{ state_account, *db, *write_session, partition->contract_kv_prefix }; + view_state.kv_ram.enable_write = true; + view_state.kv_ram.bypass_receiver_check = true; + view_state.kv_disk.enable_write = true; + view_state.kv_disk.bypass_receiver_check = true; + view_state.kv_state.enable_write = true; + uint32_t num = deltas.unpack_size(); + for (uint32_t i = 0; i < num; ++i) { + ship_protocol::table_delta delta; + deltas.unpack_next(delta); + size_t num_processed = 0; + std::visit( + [&](auto& delta_any_v) { + store_delta({ view_state }, delta_any_v, head == 0, [&]() { + if (delta_any_v.rows.size() > 10000 && !(num_processed % 10000)) { + if (shutdown()) + throw std::runtime_error("shutting down"); + ilog("block ${b} ${t} ${n} of ${r}", + ("b", block_num)("t", delta_any_v.name)("n", num_processed)("r", delta_any_v.rows.size())); + if (head == 0) { + end_write(false); + view_state.reset(); + } + } + ++num_processed; + }); + }, delta); +} + +void rodeos_db_snapshot::write_deltas(const ship_protocol::get_blocks_result_v0& result, + std::function shutdown) { + check_write(result); + if (!result.deltas) + return; + + uint32_t block_num = result.this_block->block_num; + write_deltas(block_num, eosio::opaque>(*result.deltas), shutdown); +} + +void rodeos_db_snapshot::write_deltas(const ship_protocol::get_blocks_result_v1& result, + std::function shutdown) { + check_write(result); + if (result.deltas.empty()) + return; + + uint32_t block_num = result.this_block->block_num; + write_deltas(block_num, result.deltas, shutdown); +} + +std::once_flag registered_filter_callbacks; + +rodeos_filter::rodeos_filter(eosio::name name, const std::string& wasm_filename) : name{ name } { + std::call_once(registered_filter_callbacks, filter::register_callbacks); + + std::ifstream wasm_file(wasm_filename, std::ios::binary); + if (!wasm_file.is_open()) + throw std::runtime_error("can not open " + wasm_filename); + ilog("compiling ${f}", ("f", wasm_filename)); + wasm_file.seekg(0, std::ios::end); + int len = wasm_file.tellg(); + if (len < 0) + throw std::runtime_error("wasm file length is -1"); + std::vector code(len); + wasm_file.seekg(0, std::ios::beg); + wasm_file.read((char*)code.data(), code.size()); + wasm_file.close(); + backend = std::make_unique(code, nullptr); + filter_state = std::make_unique(); + filter::rhf_t::resolve(backend->get_module()); +} + +void rodeos_filter::process(rodeos_db_snapshot& snapshot, const ship_protocol::get_blocks_result_base& result, + eosio::input_stream bin, + const std::function& push_data) { + // todo: timeout + snapshot.check_write(result); + chaindb_state chaindb_state; + db_view_state view_state{ name, *snapshot.db, *snapshot.write_session, snapshot.partition->contract_kv_prefix }; + view_state.kv_disk.enable_write = true; + view_state.kv_ram.enable_write = true; + filter::callbacks cb{ *filter_state, chaindb_state, view_state }; + filter_state->max_console_size = 10000; + filter_state->console.clear(); + filter_state->input_data = bin; + filter_state->push_data = push_data; + backend->set_wasm_allocator(&filter_state->wa); + backend->initialize(&cb); + try { + (*backend)(cb, "env", "apply", uint64_t(0), uint64_t(0), uint64_t(0)); + + if (!filter_state->console.empty()) + ilog("filter ${n} console output: <<<\n${c}>>>", ("n", name.to_string())("c", filter_state->console)); + } catch (...) { + try { + throw; + } catch ( const std::bad_alloc& ) { + throw; + } catch ( const boost::interprocess::bad_alloc& ) { + throw; + } catch( const fc::exception& e ) { + elog( "fc::exception processing filter wasm: ${e}", ("e", e.to_detail_string()) ); + } catch( const std::exception& e ) { + elog( "std::exception processing filter wasm: ${e}", ("e", e.what()) ); + } catch( ... ) { + elog( "unknown exception processing filter wasm" ); + } + if (!filter_state->console.empty()) + ilog("filter ${n} console output before exception: <<<\n${c}>>>", + ("n", name.to_string())("c", filter_state->console)); + throw; + } +} + +rodeos_query_handler::rodeos_query_handler(std::shared_ptr partition, + std::shared_ptr shared_state) + : partition{ partition }, shared_state{ std::move(shared_state) }, state_cache{ this->shared_state } {} + +} // namespace b1::rodeos diff --git a/libraries/rodeos/wasm_ql.cpp b/libraries/rodeos/wasm_ql.cpp new file mode 100644 index 00000000000..f28cc2d534e --- /dev/null +++ b/libraries/rodeos/wasm_ql.cpp @@ -0,0 +1,661 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std::literals; +namespace ship_protocol = eosio::ship_protocol; + +using boost::multi_index::indexed_by; +using boost::multi_index::member; +using boost::multi_index::multi_index_container; +using boost::multi_index::ordered_non_unique; +using boost::multi_index::sequenced; +using boost::multi_index::tag; + +using eosio::ship_protocol::action_receipt_v0; +using eosio::ship_protocol::action_trace_v1; +using eosio::ship_protocol::transaction_trace_v0; + +namespace eosio { + +// todo: abieos support for pair. Used by extensions_type. +template +void to_json(const std::pair>&, S& stream) { + eosio::check(false, eosio::convert_stream_error(stream_error::bad_variant_index)); +} + +} // namespace eosio + +namespace b1::rodeos::wasm_ql { + +template +struct overloaded : Ts... { + using Ts::operator()...; +}; +template +overloaded(Ts...)->overloaded; + +// todo: relax some of these limits +// todo: restore max_function_section_elements to 1023 and use nodeos's hard fork +struct wasm_ql_backend_options { + // static constexpr std::uint32_t max_mutable_global_bytes = 1024; + // static constexpr std::uint32_t max_table_elements = 1024; + // static constexpr std::uint32_t max_section_elements = 8191; + // static constexpr std::uint32_t max_function_section_elements = 8000; + // static constexpr std::uint32_t max_import_section_elements = 1023; + // static constexpr std::uint32_t max_element_segment_elements = 8191; + // static constexpr std::uint32_t max_data_segment_bytes = 8191; + // static constexpr std::uint32_t max_linear_memory_init = 64 * 1024; + // static constexpr std::uint32_t max_func_local_bytes = 8192; + // static constexpr std::uint32_t max_local_sets = 1023; + // static constexpr std::uint32_t eosio_max_nested_structures = 1023; + // static constexpr std::uint32_t max_br_table_elements = 8191; + // static constexpr std::uint32_t max_symbol_bytes = 8191; + // static constexpr std::uint32_t max_memory_offset = (33 * 1024 * 1024 - 1); + static constexpr std::uint32_t max_pages = 528; // 33 MiB + static constexpr std::uint32_t max_call_depth = 251; +}; + +struct callbacks; +using rhf_t = registered_host_functions; +using backend_t = eosio::vm::backend; + +struct callbacks : action_callbacks, + chaindb_callbacks, + compiler_builtins_callbacks, + console_callbacks, + context_free_system_callbacks, + db_callbacks, + memory_callbacks, + query_callbacks, + unimplemented_callbacks { + wasm_ql::thread_state& thread_state; + rodeos::chaindb_state& chaindb_state; + rodeos::db_view_state& db_view_state; + + callbacks(wasm_ql::thread_state& thread_state, rodeos::chaindb_state& chaindb_state, + rodeos::db_view_state& db_view_state) + : thread_state{ thread_state }, chaindb_state{ chaindb_state }, db_view_state{ db_view_state } {} + + auto& get_state() { return thread_state; } + auto& get_chaindb_state() { return chaindb_state; } + auto& get_db_view_state() { return db_view_state; } +}; + +std::once_flag registered_callbacks; + +void register_callbacks() { + action_callbacks::register_callbacks(); + chaindb_callbacks::register_callbacks(); + compiler_builtins_callbacks::register_callbacks(); + console_callbacks::register_callbacks(); + context_free_system_callbacks::register_callbacks(); + db_callbacks::register_callbacks(); + memory_callbacks::register_callbacks(); + query_callbacks::register_callbacks(); + unimplemented_callbacks::register_callbacks(); +} + +struct backend_entry { + eosio::name name; // only for wasms loaded from disk + eosio::checksum256 hash; // only for wasms loaded from chain + std::unique_ptr backend; +}; + +struct by_age; +struct by_name; +struct by_hash; + +using backend_container = multi_index_container< + backend_entry, + indexed_by>, // + ordered_non_unique, member>, + ordered_non_unique, member>>>; + +class backend_cache { + private: + std::mutex mutex; + const wasm_ql::shared_state& shared_state; + backend_container backends; + + public: + backend_cache(const wasm_ql::shared_state& shared_state) : shared_state{ shared_state } {} + + void add(backend_entry&& entry) { + std::lock_guard lock{ mutex }; + auto& ind = backends.get(); + ind.push_back(std::move(entry)); + while (ind.size() > shared_state.wasm_cache_size) ind.pop_front(); + } + + std::optional get(eosio::name name) { + std::optional result; + std::lock_guard lock{ mutex }; + auto& ind = backends.get(); + auto it = ind.find(name); + if (it == ind.end()) + return result; + ind.modify(it, [&](auto& x) { result = std::move(x); }); + ind.erase(it); + return result; + } + + std::optional get(const eosio::checksum256& hash) { + std::optional result; + std::lock_guard lock{ mutex }; + auto& ind = backends.get(); + auto it = ind.find(hash); + if (it == ind.end()) + return result; + ind.modify(it, [&](auto& x) { result = std::move(x); }); + ind.erase(it); + return result; + } +}; + +shared_state::shared_state(std::shared_ptr db) + : backend_cache(std::make_shared(*this)), db(std::move(db)) {} + +shared_state::~shared_state() {} + +std::optional> read_code(wasm_ql::thread_state& thread_state, eosio::name account) { + std::optional> code; + if (!thread_state.shared->contract_dir.empty()) { + auto filename = thread_state.shared->contract_dir + "/" + (std::string)account + ".wasm"; + std::ifstream wasm_file(filename, std::ios::binary); + if (wasm_file.is_open()) { + ilog("compiling ${f}", ("f", filename)); + wasm_file.seekg(0, std::ios::end); + int len = wasm_file.tellg(); + if (len < 0) + throw std::runtime_error("wasm file length is -1"); + code.emplace(len); + wasm_file.seekg(0, std::ios::beg); + wasm_file.read((char*)code->data(), code->size()); + wasm_file.close(); + } + } + return code; +} + +std::optional get_contract_hash(db_view_state& db_view_state, eosio::name account) { + std::optional result; + auto meta = get_state_row( + db_view_state.kv_state.view, + std::make_tuple(eosio::name{ "account.meta" }, eosio::name{ "primary" }, account)); + if (!meta) + return result; + auto& meta0 = std::get(meta->second); + if (!meta0.code->vm_type && !meta0.code->vm_version) + result = meta0.code->code_hash; + return result; +} + +std::optional> read_contract(db_view_state& db_view_state, const eosio::checksum256& hash, + eosio::name account) { + std::optional> result; + auto code_row = get_state_row( + db_view_state.kv_state.view, + std::make_tuple(eosio::name{ "code" }, eosio::name{ "primary" }, uint8_t(0), uint8_t(0), hash)); + if (!code_row) + return result; + auto& code0 = std::get(code_row->second); + + // todo: avoid copy + result.emplace(code0.code.pos, code0.code.end); + ilog("compiling ${h}: ${a}", ("h", eosio::convert_to_json(hash))("a", (std::string)account)); + return result; +} + +void run_action(wasm_ql::thread_state& thread_state, const std::vector& contract_kv_prefix, + ship_protocol::action& action, action_trace_v1& atrace, const rocksdb::Snapshot* snapshot, + const std::chrono::steady_clock::time_point& stop_time, std::vector>& memory) { + if (std::chrono::steady_clock::now() >= stop_time) + throw eosio::vm::timeout_exception("execution timed out"); + + chain_kv::write_session write_session{ *thread_state.shared->db, snapshot }; + db_view_state db_view_state{ state_account, *thread_state.shared->db, write_session, contract_kv_prefix }; + + std::optional entry = thread_state.shared->backend_cache->get(action.account); + std::optional> code; + if (!entry) + code = read_code(thread_state, action.account); + std::optional hash; + if (!entry && !code) { + hash = get_contract_hash(db_view_state, action.account); + if (hash) { + entry = thread_state.shared->backend_cache->get(*hash); + if (!entry) + code = read_contract(db_view_state, *hash, action.account); + } + } + + // todo: fail? silent success like normal transactions? + if (!entry && !code) + throw std::runtime_error("account " + (std::string)action.account + " has no code"); + + if (!entry) { + entry.emplace(); + if (hash) + entry->hash = *hash; + else + entry->name = action.account; + + std::call_once(registered_callbacks, register_callbacks); + entry->backend = std::make_unique(*code, nullptr); + rhf_t::resolve(entry->backend->get_module()); + } + auto se = fc::make_scoped_exit([&] { thread_state.shared->backend_cache->add(std::move(*entry)); }); + + fill_status_sing sing{ state_account, db_view_state, false }; + if (!sing.exists()) + throw std::runtime_error("No fill_status records found; is filler running?"); + auto& fill_status = sing.get(); + + // todo: move these out of thread_state since future enhancements could cause state to accidentally leak between + // queries + thread_state.max_console_size = thread_state.shared->max_console_size; + thread_state.receiver = action.account; + thread_state.action_data = action.data; + thread_state.action_return_value.clear(); + std::visit([&](auto& stat) { thread_state.block_num = stat.head; }, fill_status); + thread_state.block_info.reset(); + + chaindb_state chaindb_state; + callbacks cb{ thread_state, chaindb_state, db_view_state }; + entry->backend->set_wasm_allocator(&thread_state.wa); + + try { + eosio::vm::watchdog wd{ stop_time - std::chrono::steady_clock::now() }; + entry->backend->timed_run(wd, [&] { + entry->backend->initialize(&cb); + (*entry->backend)(cb, "env", "apply", action.account.value, action.account.value, action.name.value); + }); + } catch (...) { + atrace.console = std::move(thread_state.console); + throw; + } + + atrace.console = std::move(thread_state.console); + memory.push_back(std::move(thread_state.action_return_value)); + atrace.return_value = memory.back(); +} // run_action + +const std::vector& query_get_info(wasm_ql::thread_state& thread_state, + const std::vector& contract_kv_prefix) { + rocksdb::ManagedSnapshot snapshot{ thread_state.shared->db->rdb.get() }; + chain_kv::write_session write_session{ *thread_state.shared->db, snapshot.snapshot() }; + db_view_state db_view_state{ state_account, *thread_state.shared->db, write_session, contract_kv_prefix }; + + std::string result = "{\"server_type\":\"wasm-ql\""; + + { + global_property_kv table{ { db_view_state } }; + bool found = false; + if (table.primary_index.begin() != table.primary_index.end()) { + auto record = table.primary_index.begin().value(); + if (auto* obj = std::get_if(&record)) { + found = true; + result += ",\"chain_id\":" + eosio::convert_to_json(obj->chain_id); + } + } + if (!found) + throw std::runtime_error("No global_property_v1 records found; is filler running?"); + } + + { + fill_status_sing sing{ state_account, db_view_state, false }; + if (sing.exists()) { + std::visit( + [&](auto& obj) { + result += ",\"head_block_num\":\"" + std::to_string(obj.head) + "\""; + result += ",\"head_block_id\":" + eosio::convert_to_json(obj.head_id); + result += ",\"last_irreversible_block_num\":\"" + std::to_string(obj.irreversible) + "\""; + result += ",\"last_irreversible_block_id\":" + eosio::convert_to_json(obj.irreversible_id); + }, + sing.get()); + } else + throw std::runtime_error("No fill_status records found; is filler running?"); + } + + result += "}"; + + thread_state.action_return_value.assign(result.data(), result.data() + result.size()); + return thread_state.action_return_value; +} + +struct get_block_params { + std::string block_num_or_id = {}; +}; + +EOSIO_REFLECT(get_block_params, block_num_or_id) + +const std::vector& query_get_block(wasm_ql::thread_state& thread_state, + const std::vector& contract_kv_prefix, std::string_view body) { + get_block_params params; + std::string s{ body.begin(), body.end() }; + eosio::json_token_stream stream{ s.data() }; + try { + from_json(params, stream); + } catch (std::exception& e) { + throw std::runtime_error("An error occurred deserializing get_block_params: "s + e.what()); + } + + rocksdb::ManagedSnapshot snapshot{ thread_state.shared->db->rdb.get() }; + chain_kv::write_session write_session{ *thread_state.shared->db, snapshot.snapshot() }; + db_view_state db_view_state{ state_account, *thread_state.shared->db, write_session, contract_kv_prefix }; + + std::string bn_json = "\"" + params.block_num_or_id + "\""; + eosio::json_token_stream bn_stream{ bn_json.data() }; + + std::optional, block_info>> info; + if (params.block_num_or_id.size() == 64) { + eosio::checksum256 id; + try { + from_json(id, bn_stream); + } catch (std::exception& e) { + throw std::runtime_error("An error occurred deserializing block_num_or_id: "s + e.what()); + } + info = get_state_row_secondary(db_view_state.kv_state.view, + std::make_tuple(eosio::name{ "block.info" }, eosio::name{ "id" }, id)); + } else { + uint32_t num; + try { + from_json(num, bn_stream); + } catch (std::exception& e) { + throw std::runtime_error("An error occurred deserializing block_num_or_id: "s + e.what()); + } + info = get_state_row(db_view_state.kv_state.view, + std::make_tuple(eosio::name{ "block.info" }, eosio::name{ "primary" }, num)); + } + + if (info) { + auto& obj = std::get(info->second); + uint32_t ref_block_prefix; + memcpy(&ref_block_prefix, obj.id.value.begin() + 8, sizeof(ref_block_prefix)); + + std::string result = "{"; + result += "\"block_num\":" + eosio::convert_to_json(obj.num); + result += ",\"id\":" + eosio::convert_to_json(obj.id); + result += ",\"timestamp\":" + eosio::convert_to_json(obj.timestamp); + result += ",\"producer\":" + eosio::convert_to_json(obj.producer); + result += ",\"confirmed\":" + eosio::convert_to_json(obj.confirmed); + result += ",\"previous\":" + eosio::convert_to_json(obj.previous); + result += ",\"transaction_mroot\":" + eosio::convert_to_json(obj.transaction_mroot); + result += ",\"action_mroot\":" + eosio::convert_to_json(obj.action_mroot); + result += ",\"schedule_version\":" + eosio::convert_to_json(obj.schedule_version); + result += ",\"producer_signature\":" + eosio::convert_to_json(obj.producer_signature); + result += ",\"ref_block_prefix\":" + eosio::convert_to_json(ref_block_prefix); + result += "}"; + + thread_state.action_return_value.assign(result.data(), result.data() + result.size()); + return thread_state.action_return_value; + } + + throw std::runtime_error("block " + params.block_num_or_id + " not found"); +} // query_get_block + +struct get_abi_params { + eosio::name account_name = {}; +}; + +EOSIO_REFLECT(get_abi_params, account_name) + +struct get_abi_result { + eosio::name account_name; + std::optional abi; +}; + +EOSIO_REFLECT(get_abi_result, account_name, abi) + +const std::vector& query_get_abi(wasm_ql::thread_state& thread_state, const std::vector& contract_kv_prefix, + std::string_view body) { + get_abi_params params; + std::string s{ body.begin(), body.end() }; + eosio::json_token_stream stream{ s.data() }; + try { + from_json(params, stream); + } catch (std::exception& e) { + throw std::runtime_error("An error occurred deserializing get_abi_params: "s + e.what()); + } + + rocksdb::ManagedSnapshot snapshot{ thread_state.shared->db->rdb.get() }; + chain_kv::write_session write_session{ *thread_state.shared->db, snapshot.snapshot() }; + db_view_state db_view_state{ state_account, *thread_state.shared->db, write_session, contract_kv_prefix }; + + auto acc = get_state_row( + db_view_state.kv_state.view, + std::make_tuple(eosio::name{ "account" }, eosio::name{ "primary" }, params.account_name)); + if (!acc) + throw std::runtime_error("account " + (std::string)params.account_name + " not found"); + auto& acc0 = std::get(acc->second); + + get_abi_result result; + result.account_name = acc0.name; + if (acc0.abi.pos != acc0.abi.end) { + result.abi.emplace(); + eosio::from_bin(*result.abi, acc0.abi); + } + + // todo: avoid the extra copy + auto json = eosio::convert_to_json(result); + thread_state.action_return_value.assign(json.begin(), json.end()); + return thread_state.action_return_value; +} // query_get_abi + +// Ignores data field +struct action_no_data { + eosio::name account = {}; + eosio::name name = {}; + std::vector authorization = {}; +}; + +struct extension_hex_data { + uint16_t type = {}; + eosio::bytes data = {}; +}; + +EOSIO_REFLECT(extension_hex_data, type, data) + +EOSIO_REFLECT(action_no_data, account, name, authorization) + +struct transaction_for_get_keys : ship_protocol::transaction_header { + std::vector context_free_actions = {}; + std::vector actions = {}; + std::vector transaction_extensions = {}; +}; + +EOSIO_REFLECT(transaction_for_get_keys, base ship_protocol::transaction_header, context_free_actions, actions, + transaction_extensions) + +struct get_required_keys_params { + transaction_for_get_keys transaction = {}; + std::vector available_keys = {}; +}; + +EOSIO_REFLECT(get_required_keys_params, transaction, available_keys) + +struct get_required_keys_result { + std::vector required_keys = {}; +}; + +EOSIO_REFLECT(get_required_keys_result, required_keys) + +const std::vector& query_get_required_keys(wasm_ql::thread_state& thread_state, std::string_view body) { + get_required_keys_params params; + std::string s{ body.begin(), body.end() }; + eosio::json_token_stream stream{ s.data() }; + try { + from_json(params, stream); + } catch (std::exception& e) { + throw std::runtime_error("An error occurred deserializing get_required_keys_params: "s + e.what()); + } + + get_required_keys_result result; + for (auto& action : params.transaction.context_free_actions) + if (!action.authorization.empty()) + throw std::runtime_error("Context-free actions may not have authorizations"); + for (auto& action : params.transaction.actions) + if (!action.authorization.empty()) + throw std::runtime_error("Actions may not have authorizations"); // todo + + // todo: avoid the extra copy + auto json = eosio::convert_to_json(result); + thread_state.action_return_value.assign(json.begin(), json.end()); + return thread_state.action_return_value; +} // query_get_required_keys + +struct send_transaction_params { + std::vector signatures = {}; + std::string compression = {}; + eosio::bytes packed_context_free_data = {}; + eosio::bytes packed_trx = {}; +}; + +EOSIO_REFLECT(send_transaction_params, signatures, compression, packed_context_free_data, packed_trx) + +struct send_transaction_results { + eosio::checksum256 transaction_id; // todo: redundant with processed.id + transaction_trace_v0 processed; +}; + +EOSIO_REFLECT(send_transaction_results, transaction_id, processed) + +const std::vector& query_send_transaction(wasm_ql::thread_state& thread_state, + const std::vector& contract_kv_prefix, std::string_view body, + bool return_trace_on_except) { + send_transaction_params params; + { + std::string s{ body.begin(), body.end() }; + eosio::json_token_stream stream{ s.data() }; + try { + from_json(params, stream); + } catch (std::exception& e) { + throw std::runtime_error("An error occurred deserializing send_transaction_params: "s + e.what()); + } + } + if (params.compression != "0" && params.compression != "none") + throw std::runtime_error("Compression must be 0 or none"); // todo + ship_protocol::packed_transaction trx{ 0, + { ship_protocol::prunable_data_type::full_legacy{ + std::move(params.signatures), params.packed_context_free_data.data } }, + params.packed_trx.data }; + + rocksdb::ManagedSnapshot snapshot{ thread_state.shared->db->rdb.get() }; + + std::vector> memory; + send_transaction_results results; + results.processed = query_send_transaction(thread_state, contract_kv_prefix, trx, snapshot.snapshot(), memory, + return_trace_on_except); + + // todo: hide variants during json conversion + // todo: avoid the extra copy + auto json = eosio::convert_to_json(results); + thread_state.action_return_value.assign(json.begin(), json.end()); + return thread_state.action_return_value; +} // query_send_transaction + +bool is_signatures_empty(const ship_protocol::prunable_data_type& data) { + return std::visit(overloaded{ [](const ship_protocol::prunable_data_type::none&) { return true; }, + [](const auto& v) { return v.signatures.empty(); } }, + data.prunable_data); +} + +bool is_context_free_data_empty(const ship_protocol::prunable_data_type& data) { + return std::visit(overloaded{ [](const ship_protocol::prunable_data_type::none&) { return true; }, + [](const ship_protocol::prunable_data_type::full_legacy& v) { + return v.packed_context_free_data.pos == v.packed_context_free_data.end; + }, + [](const auto& v) { return v.context_free_segments.empty(); } }, + data.prunable_data); +} + +transaction_trace_v0 query_send_transaction(wasm_ql::thread_state& thread_state, // + const std::vector& contract_kv_prefix, // + const ship_protocol::packed_transaction& trx, // + const rocksdb::Snapshot* snapshot, // + std::vector>& memory, // + bool return_trace_on_except) { + eosio::input_stream s{ trx.packed_trx }; + ship_protocol::transaction unpacked; + try { + eosio::from_bin(unpacked, s); + } catch (std::exception& e) { throw std::runtime_error("An error occurred deserializing packed_trx: "s + e.what()); } + if (s.end != s.pos) + throw std::runtime_error("Extra data in packed_trx"); + + if (!is_signatures_empty(trx.prunable_data)) + throw std::runtime_error("Signatures must be empty"); // todo + + if (trx.compression) + throw std::runtime_error("Compression must be 0 or none"); // todo + + if (!is_context_free_data_empty(trx.prunable_data)) + throw std::runtime_error("packed_context_free_data must be empty"); + + // todo: verify query transaction extension is present, but no others + // todo: redirect if transaction extension not present? + if (!unpacked.transaction_extensions.empty()) + throw std::runtime_error("transaction_extensions must be empty"); + // todo: check expiration, ref_block_num, ref_block_prefix + if (unpacked.delay_sec.value) + throw std::runtime_error("delay_sec must be 0"); // queries can't be deferred + if (!unpacked.context_free_actions.empty()) + throw std::runtime_error("context_free_actions must be empty"); // todo: is there a case where CFA makes sense? + for (auto& action : unpacked.actions) + if (!action.authorization.empty()) + throw std::runtime_error("authorization must be empty"); // todo + + // todo: fill transaction_id + transaction_trace_v0 tt; + tt.action_traces.reserve(unpacked.actions.size()); + + auto start_time = std::chrono::steady_clock::now(); + auto stop_time = start_time + std::chrono::milliseconds{ thread_state.shared->max_exec_time_ms }; + + for (auto& action : unpacked.actions) { + tt.action_traces.emplace_back(); + auto& at = tt.action_traces.back().emplace(); + at.action_ordinal.value = tt.action_traces.size(); // starts at 1 + at.receiver = action.account; + at.act = action; + + try { + run_action(thread_state, contract_kv_prefix, action, at, snapshot, stop_time, memory); + } catch (eosio::vm::timeout_exception&) { // + throw std::runtime_error( + "timeout after " + + std::to_string(std::chrono::duration_cast(stop_time - start_time).count()) + + " ms"); + } catch (std::exception& e) { + if (!return_trace_on_except) + throw; + // todo: errorcode + at.except = tt.except = e.what(); + tt.status = ship_protocol::transaction_status::soft_fail; + break; + } + + at.receipt.emplace(); + auto& r = at.receipt->emplace(); + r.receiver = action.account; + } + + return tt; +} // query_send_transaction + +} // namespace b1::rodeos::wasm_ql diff --git a/libraries/se-helpers/CMakeLists.txt b/libraries/se-helpers/CMakeLists.txt new file mode 100644 index 00000000000..4dcc6d2ad05 --- /dev/null +++ b/libraries/se-helpers/CMakeLists.txt @@ -0,0 +1,7 @@ +if(NOT APPLE) + return() +endif() + +add_library(se-helpers se-helpers.cpp) +target_include_directories(se-helpers PUBLIC include) +target_link_libraries(se-helpers fc) \ No newline at end of file diff --git a/libraries/se-helpers/include/eosio/se-helpers/se-helpers.hpp b/libraries/se-helpers/include/eosio/se-helpers/se-helpers.hpp new file mode 100644 index 00000000000..c4d771da4f8 --- /dev/null +++ b/libraries/se-helpers/include/eosio/se-helpers/se-helpers.hpp @@ -0,0 +1,50 @@ +#pragma once + +#include +#include +#include + +#include + +#include + +namespace eosio::secure_enclave { + +class secure_enclave_key { + public: + const fc::crypto::public_key& public_key() const; + fc::crypto::signature sign(const fc::sha256& digest) const; + + secure_enclave_key(const secure_enclave_key&); + secure_enclave_key(secure_enclave_key&&); + + bool operator<(const secure_enclave_key& r) const { + return public_key() < r.public_key(); + } + + //only for use by get_all_keys()/create_key() + secure_enclave_key(void*); + + ~secure_enclave_key() = default; + private: + struct impl + { + SecKeyRef key_ref = NULL; + fc::crypto::public_key pub_key; + ~impl(); + void populate_public_key(); + }; + impl my; + + friend void delete_key(secure_enclave_key&& key); +}; + + +bool hardware_supports_secure_enclave(); +bool application_signed(); + +std::set get_all_keys(); +secure_enclave_key create_key(); +void delete_key(secure_enclave_key&& key); + +} diff --git a/libraries/se-helpers/se-helpers.cpp b/libraries/se-helpers/se-helpers.cpp new file mode 100644 index 00000000000..48127d719b3 --- /dev/null +++ b/libraries/se-helpers/se-helpers.cpp @@ -0,0 +1,236 @@ +#include + +#include +#include + +namespace eosio::secure_enclave { + +static std::string string_for_cferror(CFErrorRef error) { + CFStringRef errorString = CFCopyDescription(error); + auto release_errorString = fc::make_scoped_exit([&errorString](){CFRelease(errorString);}); + + std::string ret(CFStringGetLength(errorString), '\0'); + if(!CFStringGetCString(errorString, ret.data(), ret.size(), kCFStringEncodingUTF8)) + ret = "Unknown"; + return ret; +} + +secure_enclave_key::impl::~impl() { + if(key_ref) + CFRelease(key_ref); + key_ref = NULL; +} + +void secure_enclave_key::impl::populate_public_key() { + //without a good way to create fc::public_key direct, create a serialized version to create a public_key from + char serialized_public_key[1 + sizeof(fc::crypto::r1::public_key_data)] = {fc::get_index()}; + + SecKeyRef pubkey = SecKeyCopyPublicKey(key_ref); + + CFErrorRef error = NULL; + CFDataRef keyrep = NULL; + keyrep = SecKeyCopyExternalRepresentation(pubkey, &error); + + if(!error) { + const UInt8 *cfdata = CFDataGetBytePtr(keyrep); + memcpy(serialized_public_key + 2, cfdata + 1, 32); + serialized_public_key[1] = 0x02u + (cfdata[64] & 1u); + } + + CFRelease(keyrep); + CFRelease(pubkey); + + if(error) { + auto release_error = fc::make_scoped_exit([&error](){CFRelease(error);}); + FC_ASSERT(false, "Failed to get public key from Secure Enclave: ${m}", ("m", string_for_cferror(error))); + } + + fc::datastream ds(serialized_public_key, sizeof(serialized_public_key)); + fc::raw::unpack(ds, pub_key); +} + +secure_enclave_key::secure_enclave_key(void* seckeyref) { + my.key_ref = (SecKeyRef)(seckeyref); + my.populate_public_key(); + CFRetain(my.key_ref); +} + +secure_enclave_key::secure_enclave_key(const secure_enclave_key& o) { + my.key_ref = (SecKeyRef)CFRetain(o.my.key_ref); + my.pub_key = o.public_key(); +} + +secure_enclave_key::secure_enclave_key(secure_enclave_key&& o) { + my.key_ref = o.my.key_ref; + o.my.key_ref = NULL; + my.pub_key = o.my.pub_key; +} + +const fc::crypto::public_key &secure_enclave_key::public_key() const { + return my.pub_key; +} + +fc::crypto::signature secure_enclave_key::sign(const fc::sha256& digest) const { + fc::ecdsa_sig sig = ECDSA_SIG_new(); + CFErrorRef error = NULL; + + CFDataRef digestData = CFDataCreateWithBytesNoCopy(NULL, (UInt8*)digest.data(), digest.data_size(), kCFAllocatorNull); + CFDataRef signature = SecKeyCreateSignature(my.key_ref, kSecKeyAlgorithmECDSASignatureDigestX962SHA256, digestData, &error); + + auto cleanup = fc::make_scoped_exit([&digestData, &signature]() { + CFRelease(digestData); + if(signature) + CFRelease(signature); + }); + + if(error) { + auto release_error = fc::make_scoped_exit([&error](){CFRelease(error);}); + std::string error_string = string_for_cferror(error); + FC_ASSERT(false, "Failed to sign digest in Secure Enclave: ${m}", ("m", error_string)); + } + + const UInt8* der_bytes = CFDataGetBytePtr(signature); + long derSize = CFDataGetLength(signature); + d2i_ECDSA_SIG(&sig.obj, &der_bytes, derSize); + + char serialized_signature[sizeof(fc::crypto::r1::compact_signature) + 1] = {fc::get_index()}; + + fc::crypto::r1::compact_signature* compact_sig = (fc::crypto::r1::compact_signature *)(serialized_signature + 1); + fc::ec_key key = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1); + *compact_sig = fc::crypto::r1::signature_from_ecdsa(key, std::get(my.pub_key._storage)._data, sig, digest); + + fc::crypto::signature final_signature; + fc::datastream ds(serialized_signature, sizeof(serialized_signature)); + fc::raw::unpack(ds, final_signature); + return final_signature; +} + +secure_enclave_key create_key() { + SecAccessControlRef accessControlRef = SecAccessControlCreateWithFlags(NULL, kSecAttrAccessibleWhenUnlockedThisDeviceOnly, kSecAccessControlPrivateKeyUsage, NULL); + + int keySizeValue = 256; + CFNumberRef keySizeNumber = CFNumberCreate(NULL, kCFNumberIntType, &keySizeValue); + + const void* keyAttrKeys[] = { + kSecAttrIsPermanent, + kSecAttrAccessControl + }; + const void* keyAttrValues[] = { + kCFBooleanTrue, + accessControlRef + }; + CFDictionaryRef keyAttrDic = CFDictionaryCreate(NULL, keyAttrKeys, keyAttrValues, sizeof(keyAttrValues)/sizeof(keyAttrValues[0]), &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); + + const void* attrKeys[] = { + kSecAttrKeyType, + kSecAttrKeySizeInBits, + kSecAttrTokenID, + kSecPrivateKeyAttrs + }; + const void* atrrValues[] = { + kSecAttrKeyTypeECSECPrimeRandom, + keySizeNumber, + kSecAttrTokenIDSecureEnclave, + keyAttrDic + }; + CFDictionaryRef attributesDic = CFDictionaryCreate(NULL, attrKeys, atrrValues, sizeof(attrKeys)/sizeof(attrKeys[0]), &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); + + auto cleanup = fc::make_scoped_exit([&attributesDic, &keyAttrDic, &keySizeNumber, &accessControlRef]() { + CFRelease(attributesDic); + CFRelease(keyAttrDic); + CFRelease(keySizeNumber); + CFRelease(accessControlRef); + }); + + CFErrorRef error = NULL; + SecKeyRef privateKey = SecKeyCreateRandomKey(attributesDic, &error); + if(error) { + auto release_error = fc::make_scoped_exit([&error](){CFRelease(error);}); + FC_ASSERT(false, "Failed to create key in Secure Enclave: ${m}", ("m", string_for_cferror(error))); + } + + return secure_enclave_key(privateKey); +} + +std::set get_all_keys() { + const void* keyAttrKeys[] = { + kSecClass, + kSecAttrKeyClass, + kSecMatchLimit, + kSecReturnRef, + kSecAttrTokenID + }; + const void* keyAttrValues[] = { + kSecClassKey, + kSecAttrKeyClassPrivate, + kSecMatchLimitAll, + kCFBooleanTrue, + kSecAttrTokenIDSecureEnclave, + }; + CFDictionaryRef keyAttrDic = CFDictionaryCreate(NULL, keyAttrKeys, keyAttrValues, sizeof(keyAttrValues)/sizeof(keyAttrValues[0]), &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); + auto cleanup_keyAttrDic = fc::make_scoped_exit([&keyAttrDic](){CFRelease(keyAttrDic);}); + + std::set ret; + + CFArrayRef keyRefs = NULL; + if(SecItemCopyMatching(keyAttrDic, (CFTypeRef*)&keyRefs) || !keyRefs) + return ret; + auto cleanup_keyRefs = fc::make_scoped_exit([&keyRefs](){CFRelease(keyRefs);}); + + CFIndex count = CFArrayGetCount(keyRefs); + for(long i = 0; i < count; ++i) + ret.emplace((void*)CFArrayGetValueAtIndex(keyRefs, i)); + + return ret; +} + +void delete_key(secure_enclave_key&& key) { + CFDictionaryRef deleteDic = CFDictionaryCreate(NULL, (const void**)&kSecValueRef, (const void**)&key.my.key_ref, 1, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); + + OSStatus ret = SecItemDelete(deleteDic); + CFRelease(deleteDic); + + FC_ASSERT(ret == 0, "Failed to remove key from Secure Enclave"); +} + +bool hardware_supports_secure_enclave() { + //How to figure out if SE is available?! + char model[256]; + size_t model_size = sizeof(model); + if(sysctlbyname("hw.model", model, &model_size, nullptr, 0) == 0) { + if(strncmp(model, "iMacPro", strlen("iMacPro")) == 0) + return true; + unsigned int major, minor; + if(sscanf(model, "MacBookPro%u,%u", &major, &minor) == 2) { + if((major >= 15) || (major >= 13 && minor >= 2)) { + return true; + } + } + if(sscanf(model, "Macmini%u", &major) == 1 && major >= 8) + return true; + if(sscanf(model, "MacBookAir%u", &major) == 1 && major >= 8) + return true; + if(sscanf(model, "MacPro%u", &major) == 1 && major >= 7) + return true; + } + + return false; +} + +bool application_signed() { + OSStatus is_valid{-1}; + pid_t pid = getpid(); + SecCodeRef code = NULL; + CFNumberRef pidnumber = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pid); + CFDictionaryRef piddict = CFDictionaryCreate(kCFAllocatorDefault, (const void**)&kSecGuestAttributePid, (const void**)&pidnumber, 1, NULL, NULL); + if(!SecCodeCopyGuestWithAttributes(NULL, piddict, kSecCSDefaultFlags, &code)) { + is_valid = SecCodeCheckValidity(code, kSecCSDefaultFlags, 0); + CFRelease(code); + } + CFRelease(piddict); + CFRelease(pidnumber); + + return is_valid == errSecSuccess; +} + +} diff --git a/libraries/softfloat b/libraries/softfloat index 94dac6e56c9..44492cd324a 160000 --- a/libraries/softfloat +++ b/libraries/softfloat @@ -1 +1 @@ -Subproject commit 94dac6e56c980a99e3e38bdff89a94600de0066d +Subproject commit 44492cd324a16cf8faad854e6d8a141034c301c0 diff --git a/libraries/state_history/.clang-format b/libraries/state_history/.clang-format new file mode 100644 index 00000000000..42dd5b7832c --- /dev/null +++ b/libraries/state_history/.clang-format @@ -0,0 +1,8 @@ +BasedOnStyle: LLVM +IndentWidth: 3 +ColumnLimit: 120 +PointerAlignment: Left +AlwaysBreakTemplateDeclarations: true +AlignConsecutiveAssignments: true +AlignConsecutiveDeclarations: true +BreakConstructorInitializers: BeforeComma diff --git a/libraries/state_history/CMakeLists.txt b/libraries/state_history/CMakeLists.txt new file mode 100644 index 00000000000..e9ae6f0995b --- /dev/null +++ b/libraries/state_history/CMakeLists.txt @@ -0,0 +1,17 @@ +file(GLOB HEADERS "include/eosio/state-history/*.hpp") + +add_library( state_history + abi.cpp + create_deltas.cpp + log.cpp + transaction_trace_cache.cpp + ${HEADERS} + ) + +target_link_libraries( state_history + PUBLIC eosio_chain fc chainbase softfloat + ) + +target_include_directories( state_history + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" + ) diff --git a/plugins/state_history_plugin/state_history_plugin_abi.cpp b/libraries/state_history/abi.cpp similarity index 70% rename from plugins/state_history_plugin/state_history_plugin_abi.cpp rename to libraries/state_history/abi.cpp index 766c747d3d1..4a6b2e8a040 100644 --- a/plugins/state_history_plugin/state_history_plugin_abi.cpp +++ b/libraries/state_history/abi.cpp @@ -17,7 +17,8 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "trace_begin_block", "type": "uint32" }, { "name": "trace_end_block", "type": "uint32" }, { "name": "chain_state_begin_block", "type": "uint32" }, - { "name": "chain_state_end_block", "type": "uint32" } + { "name": "chain_state_end_block", "type": "uint32" }, + { "name": "chain_id", "type": "checksum256$" } ] }, { @@ -32,24 +33,60 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "fetch_deltas", "type": "bool" } ] }, + { + "name": "get_blocks_request_v1", "fields": [ + { "name": "start_block_num", "type": "uint32" }, + { "name": "end_block_num", "type": "uint32" }, + { "name": "max_messages_in_flight", "type": "uint32" }, + { "name": "have_positions", "type": "block_position[]" }, + { "name": "irreversible_only", "type": "bool" }, + { "name": "fetch_block", "type": "bool" }, + { "name": "fetch_traces", "type": "bool" }, + { "name": "fetch_deltas", "type": "bool" }, + { "name": "fetch_block_header", "type": "bool" } + ] + }, { "name": "get_blocks_ack_request_v0", "fields": [ { "name": "num_messages", "type": "uint32" } ] }, { - "name": "get_blocks_result_v0", "fields": [ + "name": "get_blocks_result_base", "fields": [ { "name": "head", "type": "block_position" }, { "name": "last_irreversible", "type": "block_position" }, { "name": "this_block", "type": "block_position?" }, - { "name": "prev_block", "type": "block_position?" }, + { "name": "prev_block", "type": "block_position?" } + ] + }, + { + "name": "get_blocks_result_v0", "base": "get_blocks_result_base", "fields": [ { "name": "block", "type": "bytes?" }, { "name": "traces", "type": "bytes?" }, { "name": "deltas", "type": "bytes?" } ] }, { - "name": "row", "fields": [ + "name": "get_blocks_result_v1", "base": "get_blocks_result_base", "fields": [ + { "name": "block", "type": "signed_block_variant?" }, + { "name": "traces", "type": "bytes" }, + { "name": "deltas", "type": "bytes" } + ] + }, + { + "name": "get_blocks_result_v2", "fields": [ + { "name": "head", "type": "block_position" }, + { "name": "last_irreversible", "type": "block_position" }, + { "name": "this_block", "type": "block_position?" }, + { "name": "prev_block", "type": "block_position?" }, + { "name": "block", "type": "bytes" }, + { "name": "block_header", "type": "bytes" }, + { "name": "traces", "type": "bytes" }, + { "name": "deltas", "type": "bytes" } + ] + }, + { + "name": "row_v0", "fields": [ { "name": "present", "type": "bool" }, { "name": "data", "type": "bytes" } ] @@ -57,7 +94,19 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "table_delta_v0", "fields": [ { "name": "name", "type": "string" }, - { "name": "rows", "type": "row[]" } + { "name": "rows", "type": "row_v0[]" } + ] + }, + { + "name": "row_v1", "fields": [ + { "name": "present", "type": "uint8" }, + { "name": "data", "type": "bytes" } + ] + }, + { + "name": "table_delta_v1", "fields": [ + { "name": "name", "type": "string" }, + { "name": "rows", "type": "row_v1[]" } ] }, { @@ -106,6 +155,51 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "error_code", "type": "uint64?" } ] }, + { + "name": "action_trace_v1", "fields": [ + { "name": "action_ordinal", "type": "varuint32" }, + { "name": "creator_action_ordinal", "type": "varuint32" }, + { "name": "receipt", "type": "action_receipt?" }, + { "name": "receiver", "type": "name" }, + { "name": "act", "type": "action" }, + { "name": "context_free", "type": "bool" }, + { "name": "elapsed", "type": "int64" }, + { "name": "console", "type": "string" }, + { "name": "account_ram_deltas", "type": "account_delta[]" }, + { "name": "account_disk_deltas", "type": "account_delta[]" }, + { "name": "except", "type": "string?" }, + { "name": "error_code", "type": "uint64?" }, + { "name": "return_value", "type": "bytes"} + ] + }, + { + "name": "prunable_data_none", "fields": [ + { "name": "prunable_digest", "type": "checksum256" } + ] + }, + { + "name": "prunable_data_partial", "fields": [ + { "name": "signatures", "type": "signature[]" }, + { "name": "context_free_segments", "type": "segment_type[]" } + ] + }, + { + "name": "prunable_data_full", "fields": [ + { "name": "signatures", "type": "signature[]" }, + { "name": "context_free_segments", "type": "bytes[]" } + ] + }, + { + "name": "prunable_data_full_legacy", "fields": [ + { "name": "signatures", "type": "signature[]" }, + { "name": "packed_context_free_data", "type": "bytes" } + ] + }, + { + "name": "prunable_data_type", "fields": [ + { "name": "prunable_data", "type": "prunable_data_t" } + ] + }, { "name": "partial_transaction_v0", "fields": [ { "name": "expiration", "type": "time_point_sec" }, @@ -119,6 +213,18 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "context_free_data", "type": "bytes[]" } ] }, + { + "name": "partial_transaction_v1", "fields": [ + { "name": "expiration", "type": "time_point_sec" }, + { "name": "ref_block_num", "type": "uint16" }, + { "name": "ref_block_prefix", "type": "uint32" }, + { "name": "max_net_usage_words", "type": "varuint32" }, + { "name": "max_cpu_usage_ms", "type": "uint8" }, + { "name": "delay_sec", "type": "varuint32" }, + { "name": "transaction_extensions", "type": "extension[]" }, + { "name": "prunable_data", "type": "prunable_data_type?" } + ] + }, { "name": "transaction_trace_v0", "fields": [ { "name": "id", "type": "checksum256" }, @@ -138,6 +244,13 @@ extern const char* const state_history_plugin_abi = R"({ }, { "name": "packed_transaction", "fields": [ + { "name": "compression", "type": "uint8" }, + { "name": "prunable_data", "type": "prunable_data_type" }, + { "name": "packed_trx", "type": "bytes" } + ] + }, + { + "name": "packed_transaction_v0", "fields": [ { "name": "signatures", "type": "signature[]" }, { "name": "compression", "type": "uint8" }, { "name": "packed_context_free_data", "type": "bytes" }, @@ -156,6 +269,11 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "trx", "type": "transaction_variant" } ] }, + { + "name": "transaction_receipt_v0", "base": "transaction_receipt_header", "fields": [ + { "name": "trx", "type": "transaction_variant_v0" } + ] + }, { "name": "extension", "fields": [ { "name": "type", "type": "uint16" }, @@ -181,7 +299,14 @@ extern const char* const state_history_plugin_abi = R"({ ] }, { - "name": "signed_block", "base": "signed_block_header", "fields": [ + "name": "signed_block_v0", "base": "signed_block_header", "fields": [ + { "name": "transactions", "type": "transaction_receipt_v0[]" }, + { "name": "block_extensions", "type": "extension[]" } + ] + }, + { + "name": "signed_block_v1", "base": "signed_block_header", "fields": [ + { "name": "prune_state", "type": "uint8" }, { "name": "transactions", "type": "transaction_receipt[]" }, { "name": "block_extensions", "type": "extension[]" } ] @@ -299,6 +424,14 @@ extern const char* const state_history_plugin_abi = R"({ { "type": "float128", "name": "secondary_key" } ] }, + { + "name": "key_value_v0", "fields": [ + { "type": "name", "name": "contract" }, + { "type": "bytes", "name": "key" }, + { "type": "bytes", "name": "value" }, + { "type": "name", "name": "payer" } + ] + }, { "name": "producer_key", "fields": [ { "type": "name", "name": "producer_name" }, @@ -350,6 +483,28 @@ extern const char* const state_history_plugin_abi = R"({ { "type": "uint16", "name": "max_authority_depth" } ] }, + { + "name": "chain_config_v1", "fields": [ + { "type": "uint64", "name": "max_block_net_usage" }, + { "type": "uint32", "name": "target_block_net_usage_pct" }, + { "type": "uint32", "name": "max_transaction_net_usage" }, + { "type": "uint32", "name": "base_per_transaction_net_usage" }, + { "type": "uint32", "name": "net_usage_leeway" }, + { "type": "uint32", "name": "context_free_discount_net_usage_num" }, + { "type": "uint32", "name": "context_free_discount_net_usage_den" }, + { "type": "uint32", "name": "max_block_cpu_usage" }, + { "type": "uint32", "name": "target_block_cpu_usage_pct" }, + { "type": "uint32", "name": "max_transaction_cpu_usage" }, + { "type": "uint32", "name": "min_transaction_cpu_usage" }, + { "type": "uint32", "name": "max_transaction_lifetime" }, + { "type": "uint32", "name": "deferred_trx_expiration_window" }, + { "type": "uint32", "name": "max_transaction_delay" }, + { "type": "uint32", "name": "max_inline_action_size" }, + { "type": "uint16", "name": "max_inline_action_depth" }, + { "type": "uint16", "name": "max_authority_depth" }, + { "type": "uint32", "name": "max_action_return_value_size" } + ] + }, { "name": "global_property_v0", "fields": [ { "type": "uint32?", "name": "proposed_schedule_block_num" }, @@ -497,16 +652,19 @@ extern const char* const state_history_plugin_abi = R"({ { "new_type_name": "transaction_id", "type": "checksum256" } ], "variants": [ - { "name": "request", "types": ["get_status_request_v0", "get_blocks_request_v0", "get_blocks_ack_request_v0"] }, - { "name": "result", "types": ["get_status_result_v0", "get_blocks_result_v0"] }, + { "name": "request", "types": ["get_status_request_v0", "get_blocks_request_v0", "get_blocks_ack_request_v0", "get_blocks_request_v1"] }, + { "name": "result", "types": ["get_status_result_v0", "get_blocks_result_v0", "get_blocks_result_v1", "get_blocks_result_v2"] }, { "name": "action_receipt", "types": ["action_receipt_v0"] }, - { "name": "action_trace", "types": ["action_trace_v0"] }, - { "name": "partial_transaction", "types": ["partial_transaction_v0"] }, + { "name": "action_trace", "types": ["action_trace_v0", "action_trace_v1"] }, + { "name": "partial_transaction", "types": ["partial_transaction_v0", "partial_transaction_v1"] }, { "name": "transaction_trace", "types": ["transaction_trace_v0"] }, - { "name": "transaction_variant", "types": ["transaction_id", "packed_transaction"] }, - - { "name": "table_delta", "types": ["table_delta_v0"] }, + { "name": "transaction_variant", "types": ["checksum256", "packed_transaction"] }, + { "name": "transaction_variant_v0", "types": ["transaction_id", "packed_transaction_v0"] }, + { "name": "signed_block_variant", "types": ["signed_block_v0", "signed_block_v1"] }, + { "name": "segment_type", "types": ["checksum256", "bytes"] }, + { "name": "prunable_data_t", "types": ["prunable_data_full_legacy", "prunable_data_none", "prunable_data_partial", "prunable_data_full"] }, + { "name": "table_delta", "types": ["table_delta_v0", "table_delta_v1"] }, { "name": "account", "types": ["account_v0"] }, { "name": "account_metadata", "types": ["account_metadata_v0"] }, { "name": "code", "types": ["code_v0"] }, @@ -517,7 +675,8 @@ extern const char* const state_history_plugin_abi = R"({ { "name": "contract_index256", "types": ["contract_index256_v0"] }, { "name": "contract_index_double", "types": ["contract_index_double_v0"] }, { "name": "contract_index_long_double", "types": ["contract_index_long_double_v0"] }, - { "name": "chain_config", "types": ["chain_config_v0"] }, + { "name": "key_value", "types": ["key_value_v0"] }, + { "name": "chain_config", "types": ["chain_config_v0", "chain_config_v1"] }, { "name": "global_property", "types": ["global_property_v0", "global_property_v1"] }, { "name": "generated_transaction", "types": ["generated_transaction_v0"] }, { "name": "activated_protocol_feature", "types": ["activated_protocol_feature_v0"] }, @@ -535,23 +694,24 @@ extern const char* const state_history_plugin_abi = R"({ ], "tables": [ { "name": "account", "type": "account", "key_names": ["name"] }, - { "name": "account_metadata", "type": "account_metadata", "key_names": ["name"] }, + { "name": "actmetadata", "type": "account_metadata", "key_names": ["name"] }, { "name": "code", "type": "code", "key_names": ["vm_type", "vm_version", "code_hash"] }, - { "name": "contract_table", "type": "contract_table", "key_names": ["code", "scope", "table"] }, - { "name": "contract_row", "type": "contract_row", "key_names": ["code", "scope", "table", "primary_key"] }, - { "name": "contract_index64", "type": "contract_index64", "key_names": ["code", "scope", "table", "primary_key"] }, - { "name": "contract_index128", "type": "contract_index128", "key_names": ["code", "scope", "table", "primary_key"] }, - { "name": "contract_index256", "type": "contract_index256", "key_names": ["code", "scope", "table", "primary_key"] }, - { "name": "contract_index_double", "type": "contract_index_double", "key_names": ["code", "scope", "table", "primary_key"] }, - { "name": "contract_index_long_double", "type": "contract_index_long_double", "key_names": ["code", "scope", "table", "primary_key"] }, - { "name": "global_property", "type": "global_property", "key_names": [] }, - { "name": "generated_transaction", "type": "generated_transaction", "key_names": ["sender", "sender_id"] }, - { "name": "protocol_state", "type": "protocol_state", "key_names": [] }, + { "name": "contracttbl", "type": "contract_table", "key_names": ["code", "scope", "table"] }, + { "name": "contractrow", "type": "contract_row", "key_names": ["code", "scope", "table", "primary_key"] }, + { "name": "cntrctidx1", "type": "contract_index64", "key_names": ["code", "scope", "table", "primary_key"] }, + { "name": "cntrctidx2", "type": "contract_index128", "key_names": ["code", "scope", "table", "primary_key"] }, + { "name": "cntrctidx3", "type": "contract_index256", "key_names": ["code", "scope", "table", "primary_key"] }, + { "name": "cntrctidx4", "type": "contract_index_double", "key_names": ["code", "scope", "table", "primary_key"] }, + { "name": "cntrctidx5", "type": "contract_index_long_double", "key_names": ["code", "scope", "table", "primary_key"] }, + { "name": "keyvalue", "type": "key_value", "key_names": ["contract", "key"] }, + { "name": "global.pty", "type": "global_property", "key_names": [] }, + { "name": "generatedtrx", "type": "generated_transaction", "key_names": ["sender", "sender_id"] }, + { "name": "protocolst", "type": "protocol_state", "key_names": [] }, { "name": "permission", "type": "permission", "key_names": ["owner", "name"] }, - { "name": "permission_link", "type": "permission_link", "key_names": ["account", "code", "message_type"] }, - { "name": "resource_limits", "type": "resource_limits", "key_names": ["owner"] }, - { "name": "resource_usage", "type": "resource_usage", "key_names": ["owner"] }, - { "name": "resource_limits_state", "type": "resource_limits_state", "key_names": [] }, - { "name": "resource_limits_config", "type": "resource_limits_config", "key_names": [] } + { "name": "permlink", "type": "permission_link", "key_names": ["account", "code", "message_type"] }, + { "name": "rsclimits", "type": "resource_limits", "key_names": ["owner"] }, + { "name": "rscusage", "type": "resource_usage", "key_names": ["owner"] }, + { "name": "rsclimitsst", "type": "resource_limits_state", "key_names": [] }, + { "name": "rsclimitscfg", "type": "resource_limits_config", "key_names": [] } ] })"; diff --git a/libraries/state_history/create_deltas.cpp b/libraries/state_history/create_deltas.cpp new file mode 100644 index 00000000000..463d7406dd8 --- /dev/null +++ b/libraries/state_history/create_deltas.cpp @@ -0,0 +1,213 @@ +#include +#include +#include +#include +#include + +namespace eosio { +namespace state_history { + +template +bool include_delta(const T& old, const T& curr) { + return true; +} + +bool include_delta(const chain::table_id_object& old, const chain::table_id_object& curr) { + return old.payer != curr.payer; +} + +bool include_delta(const chain::resource_limits::resource_limits_object& old, + const chain::resource_limits::resource_limits_object& curr) { + return // + old.net_weight != curr.net_weight || // + old.cpu_weight != curr.cpu_weight || // + old.ram_bytes != curr.ram_bytes; +} + +bool include_delta(const chain::resource_limits::resource_limits_state_object& old, + const chain::resource_limits::resource_limits_state_object& curr) { + return // + old.average_block_net_usage.last_ordinal != curr.average_block_net_usage.last_ordinal || // + old.average_block_net_usage.value_ex != curr.average_block_net_usage.value_ex || // + old.average_block_net_usage.consumed != curr.average_block_net_usage.consumed || // + old.average_block_cpu_usage.last_ordinal != curr.average_block_cpu_usage.last_ordinal || // + old.average_block_cpu_usage.value_ex != curr.average_block_cpu_usage.value_ex || // + old.average_block_cpu_usage.consumed != curr.average_block_cpu_usage.consumed || // + old.total_net_weight != curr.total_net_weight || // + old.total_cpu_weight != curr.total_cpu_weight || // + old.total_ram_bytes != curr.total_ram_bytes || // + old.virtual_net_limit != curr.virtual_net_limit || // + old.virtual_cpu_limit != curr.virtual_cpu_limit; +} + +bool include_delta(const chain::account_metadata_object& old, const chain::account_metadata_object& curr) { + return // + old.name != curr.name || // + old.is_privileged() != curr.is_privileged() || // + old.last_code_update != curr.last_code_update || // + old.vm_type != curr.vm_type || // + old.vm_version != curr.vm_version || // + old.code_hash != curr.code_hash; +} + +bool include_delta(const chain::code_object& old, const chain::code_object& curr) { // + return false; +} + +bool include_delta(const chain::protocol_state_object& old, const chain::protocol_state_object& curr) { + return old.activated_protocol_features != curr.activated_protocol_features; +} + +std::vector create_deltas(const chainbase::database& db, bool full_snapshot) { + std::vector deltas; + const auto& table_id_index = db.get_index(); + std::map removed_table_id; + for (auto& rem : table_id_index.last_undo_session().removed_values) + removed_table_id[rem.id._id] = &rem; + + auto get_table_id = [&](uint64_t tid) -> const chain::table_id_object& { + auto obj = table_id_index.find(tid); + if (obj) + return *obj; + auto it = removed_table_id.find(tid); + EOS_ASSERT(it != removed_table_id.end(), chain::plugin_exception, "can not found table id ${tid}", ("tid", tid)); + return *it->second; + }; + + auto pack_row = [&](auto& row) { return fc::raw::pack(make_history_serial_wrapper(db, row)); }; + auto pack_contract_row = [&](auto& row) { + return fc::raw::pack(make_history_context_wrapper(db, get_table_id(row.t_id._id), row)); + }; + + auto process_table = [&](auto* name, auto& index, auto& pack_row) { + if (full_snapshot) { + if (index.indices().empty()) + return; + deltas.push_back({}); + auto& delta = deltas.back(); + delta.name = name; + for (auto& row : index.indices()) + delta.rows.obj.emplace_back(2, pack_row(row)); + } else { + auto undo = index.last_undo_session(); + if (undo.old_values.empty() && undo.new_values.empty() && undo.removed_values.empty()) + return; + deltas.push_back({}); + auto& delta = deltas.back(); + delta.name = name; + for (auto& old : undo.old_values) { + auto& row = index.get(old.id); + if (include_delta(old, row)) + delta.rows.obj.emplace_back(1, pack_row(row)); + } + for (auto& old : undo.removed_values) + delta.rows.obj.emplace_back(0, pack_row(old)); + for (auto& row : undo.new_values) { + delta.rows.obj.emplace_back(2, pack_row(row)); + } + + if(delta.rows.obj.empty()) { + deltas.pop_back(); + } + } + }; + + process_table("account", db.get_index(), pack_row); + process_table("account_metadata", db.get_index(), pack_row); + process_table("code", db.get_index(), pack_row); + + process_table("contract_table", db.get_index(), pack_row); + process_table("contract_row", db.get_index(), pack_contract_row); + process_table("contract_index64", db.get_index(), pack_contract_row); + process_table("contract_index128", db.get_index(), pack_contract_row); + process_table("contract_index256", db.get_index(), pack_contract_row); + process_table("contract_index_double", db.get_index(), pack_contract_row); + process_table("contract_index_long_double", db.get_index(), pack_contract_row); + + process_table("key_value", db.get_index(), pack_row); + + process_table("global_property", db.get_index(), pack_row); + process_table("generated_transaction", db.get_index(), pack_row); + process_table("protocol_state", db.get_index(), pack_row); + + process_table("permission", db.get_index(), pack_row); + process_table("permission_link", db.get_index(), pack_row); + + process_table("resource_limits", db.get_index(), pack_row); + process_table("resource_usage", db.get_index(), pack_row); + process_table("resource_limits_state", db.get_index(), + pack_row); + process_table("resource_limits_config", db.get_index(), + pack_row); + + return deltas; +} + +std::vector create_deltas_rocksdb(const chainbase::database& db, const eosio::chain::kv_undo_stack_ptr &kv_undo_stack, bool full_snapshot) { + std::vector deltas; + + if(full_snapshot) { + //process key_value section + rocksdb_receiver_whole_db kv_receiver(deltas, db); + chain::backing_store::rocksdb_contract_kv_table_writer kv_writer(kv_receiver); + + auto begin_key = eosio::session::shared_bytes(&chain::backing_store::rocksdb_contract_kv_prefix, 1); + auto end_key = begin_key.next(); + chain::backing_store::walk_rocksdb_entries_with_prefix(kv_undo_stack, begin_key, end_key, kv_writer); + + //process contract section + rocksdb_receiver_whole_db db_receiver(deltas, db); + using table_collector = chain::backing_store::rocksdb_whole_db_table_collector; + table_collector table_collector_receiver(db_receiver); + chain::backing_store::rocksdb_contract_db_table_writer writer(table_collector_receiver); + + begin_key = eosio::session::shared_bytes(&chain::backing_store::rocksdb_contract_db_prefix, 1); + end_key = begin_key.next(); + chain::backing_store::walk_rocksdb_entries_with_prefix(kv_undo_stack, begin_key, end_key, writer); + } else { + auto* session = std::visit(eosio::session::overloaded{ + [](eosio::chain::kv_undo_stack_ptr::element_type::session_type* session){ + return session; + }, [](auto*){ + EOS_ASSERT(false, eosio::chain::chain_exception, "undo_stack is empty"); + static eosio::chain::kv_undo_stack_ptr::element_type::session_type* invalid = nullptr; + return invalid; + }}, kv_undo_stack->top().holder()); + + rocksdb_receiver_single_entry receiver(deltas, db); + + for(auto &updated_key: session->updated_keys()) { + std::visit([&](auto* p) { + p->read(updated_key) ? receiver.set_delta_present(1) : receiver.set_delta_present(2); + }, session->parent()); + + chain::backing_store::process_rocksdb_entry(*session, updated_key, receiver); + } + + receiver.set_delta_present(0); + for(auto &deleted_key: session->deleted_keys()) { + std::visit([&](auto* p) { + chain::backing_store::process_rocksdb_entry(*p, deleted_key, receiver); + }, session->parent()); + } + } + + return deltas; +} + +std::vector create_deltas(const chain::combined_database& db, bool full_snapshot){ + auto &chainbase_db = db.get_db(); + auto &kv_undo_stack = db.get_kv_undo_stack(); + + std::vector deltas = create_deltas(chainbase_db, full_snapshot); + + if(kv_undo_stack && chainbase_db.get().backing_store == chain::backing_store_type::ROCKSDB) { + auto deltas_rocksdb = create_deltas_rocksdb(chainbase_db, kv_undo_stack, full_snapshot); + deltas.insert( deltas.end(), deltas_rocksdb.begin(), deltas_rocksdb.end() ); + } + + return deltas; +} + +} // namespace state_history +} // namespace eosio diff --git a/libraries/state_history/include/eosio/state_history/compression.hpp b/libraries/state_history/include/eosio/state_history/compression.hpp new file mode 100644 index 00000000000..72611abe61a --- /dev/null +++ b/libraries/state_history/include/eosio/state_history/compression.hpp @@ -0,0 +1,86 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace eosio { +namespace state_history { + +namespace bio = boost::iostreams; +template +struct length_writer { + STREAM& strm; + uint64_t start_pos = 0; + + length_writer(STREAM& f) + : strm(f) { + uint32_t len = 0; + strm.write((const char*)&len, sizeof(len)); + start_pos = strm.tellp(); + } + + ~length_writer() { + uint64_t end_pos = strm.tellp(); + uint32_t len = end_pos - start_pos; + strm.seekp(start_pos - sizeof(len)); + strm.write((char*)&len, sizeof(len)); + strm.seekp(end_pos); + } +}; + +template +bool is_empty(const T&) { + return false; +} +template +bool is_empty(const std::vector& obj) { + return obj.empty(); +} + +template +void zlib_pack(STREAM& strm, const T& obj) { + if (is_empty(obj)) { + fc::raw::pack(strm, uint32_t(0)); + } + else { + length_writer len_writer(strm); + fc::datastream compressed_strm(bio::zlib_compressor() | fc::to_sink(strm)); + fc::raw::pack(compressed_strm, obj); + } +} + +template +void zlib_unpack(STREAM& strm, T& obj) { + uint32_t len; + fc::raw::unpack(strm, len); + auto pos = strm.tellp(); + if (len > 0) { + fc::datastream decompress_strm(bio::zlib_decompressor() | bio::restrict(fc::to_source(strm), 0, len)); + fc::raw::unpack(decompress_strm, obj); + // zlib may add some padding at the end of the uncompressed data so that the position of `strm` wont be at the start of next entry, + // we need to use `seek()` to adjust the position of `strm`. + strm.seekp(pos + len); + } +} + +template +std::vector zlib_decompress(STREAM& strm) { + uint32_t len; + fc::raw::unpack(strm, len); + if (len > 0) { + std::vector result; + bio::filtering_istreambuf decompress_buf(bio::zlib_decompressor() | bio::restrict(fc::to_source(strm), 0, len)); + bio::copy(decompress_buf, bio::back_inserter(result)); + return result; + } + return {}; +} + +} // namespace state_history +} // namespace eosio diff --git a/libraries/state_history/include/eosio/state_history/create_deltas.hpp b/libraries/state_history/include/eosio/state_history/create_deltas.hpp new file mode 100644 index 00000000000..ff47874c4f8 --- /dev/null +++ b/libraries/state_history/include/eosio/state_history/create_deltas.hpp @@ -0,0 +1,12 @@ +#pragma once + +#include +#include + +namespace eosio { +namespace state_history { + +std::vector create_deltas(const chain::combined_database& db, bool full_snapshot); + +} // namespace state_history +} // namespace eosio diff --git a/libraries/state_history/include/eosio/state_history/log.hpp b/libraries/state_history/include/eosio/state_history/log.hpp new file mode 100644 index 00000000000..595628da155 --- /dev/null +++ b/libraries/state_history/include/eosio/state_history/log.hpp @@ -0,0 +1,227 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace eosio { + +namespace bfs = boost::filesystem; + +/* + * *.log: + * +---------+----------------+-----------+------------------+-----+---------+----------------+ + * | Entry i | Pos of Entry i | Entry i+1 | Pos of Entry i+1 | ... | Entry z | Pos of Entry z | + * +---------+----------------+-----------+------------------+-----+---------+----------------+ + * + * *.index: + * +----------------+------------------+-----+----------------+ + * | Pos of Entry i | Pos of Entry i+1 | ... | Pos of Entry z | + * +----------------+------------------+-----+----------------+ + * + * each entry: + * state_history_log_header + * payload + */ + +inline uint64_t ship_magic(uint32_t version) { + using namespace eosio::chain::literals; + return "ship"_n.to_uint64_t() | version; +} +inline bool is_ship(uint64_t magic) { + using namespace eosio::chain::literals; + return (magic & 0xffff'ffff'0000'0000) == "ship"_n.to_uint64_t(); +} +inline uint32_t get_ship_version(uint64_t magic) { return magic; } +inline bool is_ship_supported_version(uint64_t magic) { return get_ship_version(magic) <= 1; } +static const uint32_t ship_current_version = 1; + +struct state_history_log_header { + uint64_t magic = ship_magic(ship_current_version); + chain::block_id_type block_id = {}; + uint64_t payload_size = 0; +}; +static const int state_history_log_header_serial_size = sizeof(state_history_log_header::magic) + + sizeof(state_history_log_header::block_id) + + sizeof(state_history_log_header::payload_size); + +class state_history_log_data : public chain::log_data_base { + std::string filename; + + public: + state_history_log_data() = default; + state_history_log_data(const fc::path& path, mapmode mode = mapmode::readonly) + : filename(path.string()) { + open(path, mode); + } + + void open(const fc::path& path, mapmode mode = mapmode::readonly) { + if (file.is_open()) + file.close(); + file.open(path.string(), mode); + return; + } + + uint32_t version() const { return get_ship_version(chain::read_buffer(file.const_data())); } + uint32_t first_block_num() const { return block_num_at(0); } + uint32_t first_block_position() const { return 0; } + + std::pair, uint32_t> ro_stream_at(uint64_t pos) const { + uint32_t ver = get_ship_version(chain::read_buffer(file.const_data() + pos)); + return std::make_pair( + fc::datastream(file.const_data() + pos + sizeof(state_history_log_header), payload_size_at(pos)), + ver); + } + + std::pair, uint32_t> rw_stream_at(uint64_t pos) const { + uint32_t ver = get_ship_version(chain::read_buffer(file.const_data() + pos)); + return std::make_pair( + fc::datastream(file.data() + pos + sizeof(state_history_log_header), payload_size_at(pos)), ver); + } + + uint32_t block_num_at(uint64_t position) const { + return fc::endian_reverse_u32( + chain::read_buffer(file.const_data() + position + offsetof(state_history_log_header, block_id))); + } + + chain::block_id_type block_id_at(uint64_t position) const { + return chain::read_buffer(file.const_data() + position + + offsetof(state_history_log_header, block_id)); + } + + uint64_t payload_size_at(uint64_t pos) const; + void construct_index(const fc::path& index_file_name) const; +}; + +struct state_history_config { + bfs::path log_dir; + bfs::path retained_dir; + bfs::path archive_dir; + uint32_t stride = UINT32_MAX; + uint32_t max_retained_files = 10; +}; + +class state_history_log { + private: + using cfile_stream = fc::datastream; + const char* const name = ""; + cfile_stream index; + uint32_t _begin_block = 0; + uint32_t _end_block = 0; + chain::block_id_type last_block_id; + uint32_t version = ship_current_version; + uint32_t stride; + + protected: + cfile_stream write_log; + cfile_stream read_log; + + using catalog_t = chain::log_catalog>; + catalog_t catalog; + + public: + // The type aliases below help to make it obvious about the meanings of member function return values. + using block_num_type = uint32_t; + using version_type = uint32_t; + using file_position_type = uint64_t; + using config_type = state_history_config; + + state_history_log(const char* const name, const state_history_config& conf); + + block_num_type begin_block() const { + block_num_type result = catalog.first_block_num(); + return result != 0 ? result : _begin_block; + } + block_num_type end_block() const { return _end_block; } + + template + void write_entry(state_history_log_header& header, const chain::block_id_type& prev_id, F write_payload) { + + auto [block_num, start_pos] = write_entry_header(header, prev_id); + try { + write_payload(write_log); + write_entry_position(header, start_pos, block_num); + } catch (...) { + write_log.close(); + boost::filesystem::resize_file(write_log.get_file_path(), start_pos); + write_log.open("rb+"); + throw; + } + } + + std::optional get_block_id(block_num_type block_num); + + protected: + void get_entry_header(block_num_type block_num, state_history_log_header& header); + + private: + void read_header(state_history_log_header& header, bool assert_version = true); + void write_header(const state_history_log_header& header); + bool get_last_block(uint64_t size); + void recover_blocks(uint64_t size); + void open_log(bfs::path filename); + void open_index(bfs::path filename); + file_position_type get_pos(block_num_type block_num); + void truncate(block_num_type block_num); + void split_log(); + + /** + * @returns the block num and the file position + **/ + std::pair write_entry_header(const state_history_log_header& header, const chain::block_id_type& prev_id); + void write_entry_position(const state_history_log_header& header, file_position_type pos, block_num_type block_num); +}; // state_history_log + +class state_history_traces_log : public state_history_log { + state_history::transaction_trace_cache cache; + + public: + bool trace_debug_mode = false; + state_history::compression_type compression = state_history::compression_type::zlib; + + state_history_traces_log(const state_history_config& conf); + + static bool exists(bfs::path state_history_dir); + + void add_transaction(const chain::transaction_trace_ptr& trace, const chain::packed_transaction_ptr& transaction) { + cache.add_transaction(trace, transaction); + } + + chain::bytes get_log_entry(block_num_type block_num); + + void block_start(uint32_t block_num) { cache.clear(); } + + void store(const chainbase::database& db, const chain::block_state_ptr& block_state); + + /** + * @param[in,out] ids The ids to been pruned and returns the ids not found in the specified block + **/ + void prune_transactions(block_num_type block_num, std::vector& ids); +}; + +class state_history_chain_state_log : public state_history_log { + public: + state_history_chain_state_log(const state_history_config& conf); + + chain::bytes get_log_entry(block_num_type block_num); + + void store(const chain::combined_database& db, const chain::block_state_ptr& block_state); +}; + +} // namespace eosio + +FC_REFLECT(eosio::state_history_log_header, (magic)(block_id)(payload_size)) diff --git a/libraries/state_history/include/eosio/state_history/rocksdb_receiver.hpp b/libraries/state_history/include/eosio/state_history/rocksdb_receiver.hpp new file mode 100644 index 00000000000..9dc1503be84 --- /dev/null +++ b/libraries/state_history/include/eosio/state_history/rocksdb_receiver.hpp @@ -0,0 +1,186 @@ +#pragma once + +#include +#include + +namespace eosio { +namespace state_history { + +/* + * Base class for receivers of rocksdb data for SHiP delta generation. + * +*/ +class rocksdb_receiver { +public: + // This method is not required for this use case, but the interface requires we implement it. + void add_row(fc::unsigned_int x) { + + } + +protected: + rocksdb_receiver(std::vector &deltas, const chainbase::database& db) : deltas_(deltas), db_(db) {}; + + std::optional table_; + uint8_t present_{2}; + + template + void add_delta_with_table(const char *name, const Object& row){ + deltas_[get_delta_index(name)].rows.obj.emplace_back(present_, fc::raw::pack(make_history_context_wrapper(db_, *table_, row))); + } + + template + void add_delta(const char *name, const Object& row){ + deltas_[get_delta_index(name)].rows.obj.emplace_back(present_, fc::raw::pack(make_history_serial_wrapper(db_, row))); + } + +private: + std::map name_to_index_; + + int get_delta_index(std::string name) { + if(!name_to_index_.count(name)) { + deltas_.push_back({}); + deltas_.back().name = name; + name_to_index_[name] = deltas_.size() - 1; + } + + return name_to_index_[name]; + } + + std::vector &deltas_; + const chainbase::database& db_; + +}; + +/* + * Specialization of rocksdb data's receiver for SHiP delta generation when we are getting the whole db. + * + * When we are receiving the whole current db content, then we store the deltas of the objects as we + * receive them. + */ +class rocksdb_receiver_whole_db : public rocksdb_receiver { +public: + using rocksdb_receiver::add_row; + + rocksdb_receiver_whole_db(std::vector &deltas, const chainbase::database& db) : rocksdb_receiver(deltas, db) {} + + void add_row(const chain::kv_object_view& row) { + add_delta("key_value", row); + } + + void add_row(const chain::backing_store::table_id_object_view& row) { + add_delta("contract_table", row); + table_ = row; + } + + void add_row(const chain::backing_store::primary_index_view& row) { + add_delta_with_table("contract_row", row); + } + + void add_row(const chain::backing_store::secondary_index_view& row) { + add_delta_with_table("contract_index64", row); + } + + void add_row(const chain::backing_store::secondary_index_view& row) { + add_delta_with_table("contract_index128", row); + } + + void add_row(const chain::backing_store::secondary_index_view& row) { + add_delta_with_table("contract_index256", row); + } + + void add_row(const chain::backing_store::secondary_index_view& row) { + add_delta_with_table("contract_index_double", row); + } + + void add_row(const chain::backing_store::secondary_index_view& row) { + add_delta_with_table("contract_index_long_double", row); + } +}; + + +/* + * Specialization of rocksdb data's receiver for SHiP delta generation when we are operating on single key entries (updated or deleted). + * + * This will be used when working with single key entries, because some of these objects (primary and secondary indexes) + * require information about the table they belong to (the context table), then for these objects we will receive 2 calls, + * the first one related to the table_id_object_view (but in this case this object does not contain proper values for + * payer so it can't be used to create its delta) associated with them andthe second call for the object itself. + * For that reason there is logic also associated to detect when we want to store the table_id_object_view itself + * (which is when we receive only 1 call for this and the next call is not any of the primary or secondary indexes). + * +*/ +class rocksdb_receiver_single_entry : public rocksdb_receiver { +public: + using rocksdb_receiver::add_row; + + rocksdb_receiver_single_entry(std::vector &deltas, const chainbase::database& db) : rocksdb_receiver(deltas, db) {} + + void add_row(const chain::kv_object_view& row) { + maybe_process_table(); + + add_delta("key_value", row); + } + + /* + * In this case we do not immediately report it as a delta until we see what comes next. If it is followed by a row + * that requires the table context (primary and secondary indexes) then the table_id_object_view was just being + * provided as the context. If it is followed by any other case, then it was indicating a change in that + * table_id_object_view, and it needs to be added to the delta. + */ + void add_row(const chain::backing_store::table_id_object_view& row) { + maybe_process_table(); + table_ = row; + } + + void add_row(const chain::backing_store::primary_index_view& row) { + add_delta_with_table("contract_row", row); + table_ = {}; + } + + void add_row(const chain::backing_store::secondary_index_view& row) { + add_delta_with_table("contract_index64", row); + table_ = {}; + } + + void add_row(const chain::backing_store::secondary_index_view& row) { + add_delta_with_table("contract_index128", row); + table_ = {}; + } + + void add_row(const chain::backing_store::secondary_index_view& row) { + add_delta_with_table("contract_index256", row); + table_ = {}; + } + + void add_row(const chain::backing_store::secondary_index_view& row) { + add_delta_with_table("contract_index_double", row); + table_ = {}; + } + + void add_row(const chain::backing_store::secondary_index_view& row) { + add_delta_with_table("contract_index_long_double", row); + table_ = {}; + } + + void set_delta_present(uint8_t value) { + maybe_process_table(); + + present_ = value; + } + + ~rocksdb_receiver_single_entry() { + maybe_process_table(); + } + +private: + + void maybe_process_table() { + if(table_) { + add_delta("contract_table", *table_); + table_ = {}; + } + } +}; + +} +} diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp b/libraries/state_history/include/eosio/state_history/serialization.hpp similarity index 57% rename from plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp rename to libraries/state_history/include/eosio/state_history/serialization.hpp index f7085a4568e..f6522422593 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp +++ b/libraries/state_history/include/eosio/state_history/serialization.hpp @@ -1,18 +1,20 @@ #pragma once #include +#include #include #include #include #include +#include #include #include #include #include #include #include -#include -#include +#include +#include #include @@ -35,11 +37,16 @@ struct history_context_wrapper { }; template -history_context_wrapper, std::decay_t> -make_history_context_wrapper(const chainbase::database& db, const P& context, const T& obj) { +history_context_wrapper, std::decay_t> make_history_context_wrapper(const chainbase::database& db, + const P& context, const T& obj) { return {db, context, obj}; } +struct trace_receipt_context { + uint8_t failed_status = eosio::chain::transaction_receipt_header::hard_fail; + bool debug_mode = false; +}; + namespace fc { template @@ -48,7 +55,7 @@ const T& as_type(const T& x) { } template -datastream& history_serialize_container(datastream& ds, const chainbase::database& db, const T& v) { +ST& history_serialize_container(ST& ds, const chainbase::database& db, const T& v) { fc::raw::pack(ds, unsigned_int(v.size())); for (auto& x : v) ds << make_history_serial_wrapper(db, x); @@ -56,8 +63,7 @@ datastream& history_serialize_container(datastream& ds, const chainbase: } template -datastream& history_serialize_container(datastream& ds, const chainbase::database& db, - const std::vector>& v) { +ST& history_serialize_container(ST& ds, const chainbase::database& db, const std::vector>& v) { fc::raw::pack(ds, unsigned_int(v.size())); for (auto& x : v) { EOS_ASSERT(!!x, eosio::chain::plugin_exception, "null inside container"); @@ -67,8 +73,8 @@ datastream& history_serialize_container(datastream& ds, const chainbase: } template -datastream& history_context_serialize_container(datastream& ds, const chainbase::database& db, const P& context, - const std::vector& v) { +ST& history_context_serialize_container(ST& ds, const chainbase::database& db, const P& context, + const std::vector& v) { fc::raw::pack(ds, unsigned_int(v.size())); for (const auto& x : v) { ds << make_history_context_wrapper(db, context, x); @@ -77,7 +83,7 @@ datastream& history_context_serialize_container(datastream& ds, const ch } template -datastream& operator<<(datastream& ds, const history_serial_big_vector_wrapper& obj) { +ST& operator<<(ST& ds, const eosio::state_history::big_vector_wrapper& obj) { FC_ASSERT(obj.obj.size() <= 1024 * 1024 * 1024); fc::raw::pack(ds, unsigned_int((uint32_t)obj.obj.size())); for (auto& x : obj.obj) @@ -85,49 +91,25 @@ datastream& operator<<(datastream& ds, const history_serial_big_vector_w return ds; } -template -inline void history_pack_varuint64(datastream& ds, uint64_t val) { - do { - uint8_t b = uint8_t(val) & 0x7f; - val >>= 7; - b |= ((val > 0) << 7); - ds.write((char*)&b, 1); - } while (val); -} - -template -void history_pack_big_bytes(datastream& ds, const eosio::chain::bytes& v) { - history_pack_varuint64(ds, v.size()); - if (v.size()) - ds.write(&v.front(), (uint32_t)v.size()); -} - -template -void history_pack_big_bytes(datastream& ds, const fc::optional& v) { - fc::raw::pack(ds, v.valid()); - if (v) - history_pack_big_bytes(ds, *v); -} - template -datastream& operator<<(datastream& ds, const history_serial_wrapper>& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper>& obj) { return history_serialize_container(ds, obj.db, obj.obj); } template -datastream& operator<<(datastream& ds, const history_context_wrapper>& obj) { +ST& operator<<(ST& ds, const history_context_wrapper>& obj) { return history_context_serialize_container(ds, obj.db, obj.context, obj.obj); } template -datastream& operator<<(datastream& ds, const history_serial_wrapper>& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper>& obj) { fc::raw::pack(ds, obj.obj.first); fc::raw::pack(ds, obj.obj.second); return ds; } template -datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.name.to_uint64_t())); fc::raw::pack(ds, as_type(obj.obj.creation_date)); @@ -136,8 +118,7 @@ datastream& operator<<(datastream& ds, const history_serial_wrapper -datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.name.to_uint64_t())); fc::raw::pack(ds, as_type(obj.obj.is_privileged())); @@ -153,7 +134,7 @@ datastream& operator<<(datastream& } template -datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.vm_type)); fc::raw::pack(ds, as_type(obj.obj.vm_version)); @@ -162,8 +143,8 @@ datastream& operator<<(datastream& ds, const history_serial_wrapper -datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { +template +ST& serialize_table_id_object(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.code.to_uint64_t())); fc::raw::pack(ds, as_type(obj.obj.scope.to_uint64_t())); @@ -173,9 +154,18 @@ datastream& operator<<(datastream& ds, const history_serial_wrapper -datastream& -operator<<(datastream& ds, - const history_context_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { + return serialize_table_id_object(ds, obj); +} + +template +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { + return serialize_table_id_object(ds, obj); +} + +template +ST& operator<<(ST& ds, + const history_context_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.context.code.to_uint64_t())); fc::raw::pack(ds, as_type(obj.context.scope.to_uint64_t())); @@ -186,27 +176,40 @@ operator<<(datastream& return ds; } +template +ST& operator<<(ST& ds, + const history_context_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.context.code.to_uint64_t())); + fc::raw::pack(ds, as_type(obj.context.scope.to_uint64_t())); + fc::raw::pack(ds, as_type(obj.context.table.to_uint64_t())); + fc::raw::pack(ds, as_type(obj.obj.primary_key)); + fc::raw::pack(ds, as_type(obj.obj.payer.to_uint64_t())); + fc::raw::pack(ds, obj.obj.value); + return ds; +} + template -void serialize_secondary_index_data(datastream& ds, const T& obj) { +void serialize_secondary_index_data(ST& ds, const T& obj) { fc::raw::pack(ds, obj); } template -void serialize_secondary_index_data(datastream& ds, const float64_t& obj) { +void serialize_secondary_index_data(ST& ds, const float64_t& obj) { uint64_t i; memcpy(&i, &obj, sizeof(i)); fc::raw::pack(ds, i); } template -void serialize_secondary_index_data(datastream& ds, const float128_t& obj) { +void serialize_secondary_index_data(ST& ds, const float128_t& obj) { __uint128_t i; memcpy(&i, &obj, sizeof(i)); fc::raw::pack(ds, i); } template -void serialize_secondary_index_data(datastream& ds, const eosio::chain::key256_t& obj) { +void serialize_secondary_index_data(ST& ds, const eosio::chain::key256_t& obj) { auto rev = [&](__uint128_t x) { char* ch = reinterpret_cast(&x); std::reverse(ch, ch + sizeof(x)); @@ -216,9 +219,8 @@ void serialize_secondary_index_data(datastream& ds, const eosio::chain::key2 fc::raw::pack(ds, rev(obj[1])); } -template -datastream& serialize_secondary_index(datastream& ds, const eosio::chain::table_id_object& context, - const T& obj) { +template +ST& serialize_secondary_index(ST& ds, const TableIdObject& context, const T& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(context.code.to_uint64_t())); fc::raw::pack(ds, as_type(context.scope.to_uint64_t())); @@ -230,68 +232,110 @@ datastream& serialize_secondary_index(datastream& ds, const eosio::chain } template -datastream& -operator<<(datastream& ds, - const history_context_wrapper& obj) { +ST& operator<<(ST& ds, + const history_context_wrapper& obj) { return serialize_secondary_index(ds, obj.context, obj.obj); } template -datastream& -operator<<(datastream& ds, - const history_context_wrapper& obj) { +ST& operator<<(ST& ds, + const history_context_wrapper>& obj) { return serialize_secondary_index(ds, obj.context, obj.obj); } template -datastream& -operator<<(datastream& ds, - const history_context_wrapper& obj) { +ST& operator<<(ST& ds, + const history_context_wrapper& obj) { return serialize_secondary_index(ds, obj.context, obj.obj); } template -datastream& -operator<<(datastream& ds, - const history_context_wrapper& obj) { +ST& operator<<(ST& ds, + const history_context_wrapper>& obj) { return serialize_secondary_index(ds, obj.context, obj.obj); } template -datastream& operator<<( - datastream& ds, - const history_context_wrapper& obj) { +ST& operator<<(ST& ds, + const history_context_wrapper& obj) { return serialize_secondary_index(ds, obj.context, obj.obj); } template -datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, + const history_context_wrapper>& obj) { + return serialize_secondary_index(ds, obj.context, obj.obj); +} + +template +ST& operator<<(ST& ds, + const history_context_wrapper& obj) { + return serialize_secondary_index(ds, obj.context, obj.obj); +} + +template +ST& operator<<(ST& ds, + const history_context_wrapper>& obj) { + return serialize_secondary_index(ds, obj.context, obj.obj); +} + +template +ST& operator<<( + ST& ds, const history_context_wrapper& obj) { + return serialize_secondary_index(ds, obj.context, obj.obj); +} + +template +ST& operator<<(ST& ds, + const history_context_wrapper>& obj) { + return serialize_secondary_index(ds, obj.context, obj.obj); +} + +template +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.contract.to_uint64_t())); + fc::raw::pack(ds, as_type(obj.obj.kv_key)); + fc::raw::pack(ds, as_type(obj.obj.kv_value)); + fc::raw::pack(ds, as_type(obj.obj.payer.to_uint64_t())); + return ds; +} + +template +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(0)); + fc::raw::pack(ds, as_type(obj.obj.contract.to_uint64_t())); + fc::raw::pack(ds, as_type(obj.obj.kv_key)); + fc::raw::pack(ds, as_type(obj.obj.kv_value)); + fc::raw::pack(ds, as_type(obj.obj.payer.to_uint64_t())); + return ds; +} + +template +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, as_type(obj.obj.threshold)); history_serialize_container(ds, obj.db, as_type>(obj.obj.keys)); - } template -datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, as_type(obj.obj.producer_name.to_uint64_t())); fc::raw::pack(ds, as_type(obj.obj.authority)); return ds; } template -datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, as_type(obj.obj.version)); - history_serialize_container(ds, obj.db, - as_type>(obj.obj.producers)); + history_serialize_container( + ds, obj.db, as_type>(obj.obj.producers)); return ds; } template -datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { + fc::raw::pack(ds, fc::unsigned_int(1)); fc::raw::pack(ds, as_type(obj.obj.max_block_net_usage)); fc::raw::pack(ds, as_type(obj.obj.target_block_net_usage_pct)); fc::raw::pack(ds, as_type(obj.obj.max_transaction_net_usage)); @@ -309,14 +353,14 @@ datastream& operator<<(datastream& ds, const history_serial_wrapper(obj.obj.max_inline_action_size)); fc::raw::pack(ds, as_type(obj.obj.max_inline_action_depth)); fc::raw::pack(ds, as_type(obj.obj.max_authority_depth)); + fc::raw::pack(ds, as_type(obj.obj.max_action_return_value_size)); return ds; } template -datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(1)); - fc::raw::pack(ds, as_type>(obj.obj.proposed_schedule_block_num)); + fc::raw::pack(ds, as_type>(obj.obj.proposed_schedule_block_num)); fc::raw::pack(ds, make_history_serial_wrapper( obj.db, as_type(obj.obj.proposed_schedule))); fc::raw::pack(ds, make_history_serial_wrapper(obj.db, as_type(obj.obj.configuration))); @@ -326,8 +370,7 @@ datastream& operator<<(datastream& } template -datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.sender.to_uint64_t())); fc::raw::pack(ds, as_type<__uint128_t>(obj.obj.sender_id)); @@ -338,9 +381,8 @@ datastream& operator<<(datastream& } template -datastream& -operator<<(datastream& ds, - const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, + const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.feature_digest)); fc::raw::pack(ds, as_type(obj.obj.activation_block_num)); @@ -348,43 +390,42 @@ operator<<(datastream& } template -datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); history_serialize_container(ds, obj.db, obj.obj.activated_protocol_features); return ds; } template -datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, as_type(obj.obj.key)); fc::raw::pack(ds, as_type(obj.obj.weight)); return ds; } template -datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, as_type(obj.obj.actor.to_uint64_t())); fc::raw::pack(ds, as_type(obj.obj.permission.to_uint64_t())); return ds; } template -datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, make_history_serial_wrapper(obj.db, as_type(obj.obj.permission))); fc::raw::pack(ds, as_type(obj.obj.weight)); return ds; } template -datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, as_type(obj.obj.wait_sec)); fc::raw::pack(ds, as_type(obj.obj.weight)); return ds; } template -datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, as_type(obj.obj.threshold)); history_serialize_container(ds, obj.db, obj.obj.keys); history_serialize_container(ds, obj.db, obj.obj.accounts); @@ -393,7 +434,7 @@ datastream& operator<<(datastream& ds, const history_serial_wrapper -datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.owner.to_uint64_t())); fc::raw::pack(ds, as_type(obj.obj.name.to_uint64_t())); @@ -401,11 +442,12 @@ datastream& operator<<(datastream& ds, const history_serial_wrapper(); const auto* parent = index.find(obj.obj.parent); if (!parent) { - auto& undo = index.stack().back(); - auto it = undo.removed_values.find(obj.obj.parent); + auto undo = index.last_undo_session(); + auto it = std::find_if(undo.removed_values.begin(), undo.removed_values.end(), + [&](auto& x) { return x.id._id == obj.obj.parent; }); EOS_ASSERT(it != undo.removed_values.end(), eosio::chain::plugin_exception, "can not find parent of permission_object"); - parent = &it->second; + parent = &*it; } fc::raw::pack(ds, as_type(parent->name.to_uint64_t())); } else { @@ -417,8 +459,7 @@ datastream& operator<<(datastream& ds, const history_serial_wrapper -datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.account.to_uint64_t())); fc::raw::pack(ds, as_type(obj.obj.code.to_uint64_t())); @@ -428,8 +469,7 @@ datastream& operator<<(datastream& } template -datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { EOS_ASSERT(!obj.obj.pending, eosio::chain::plugin_exception, "accepted_block sent while resource_limits_object in pending state"); fc::raw::pack(ds, fc::unsigned_int(0)); @@ -441,8 +481,7 @@ datastream& operator<<(datastream& } template -datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.last_ordinal)); fc::raw::pack(ds, as_type(obj.obj.value_ex)); @@ -451,8 +490,7 @@ datastream& operator<<(datastream& } template -datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.owner.to_uint64_t())); fc::raw::pack(ds, make_history_serial_wrapper( @@ -464,9 +502,7 @@ datastream& operator<<(datastream& } template -datastream& -operator<<(datastream& ds, - const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, make_history_serial_wrapper(obj.db, as_type( obj.obj.average_block_net_usage))); @@ -481,8 +517,7 @@ operator<<(datastream& } template -datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.numerator)); fc::raw::pack(ds, as_type(obj.obj.denominator)); @@ -490,8 +525,7 @@ datastream& operator<<(datastream& } template -datastream& operator<<(datastream& ds, - const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.target)); fc::raw::pack(ds, as_type(obj.obj.max)); @@ -505,9 +539,8 @@ datastream& operator<<(datastream& } template -datastream& -operator<<(datastream& ds, - const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, + const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack( ds, make_history_serial_wrapper( @@ -521,38 +554,39 @@ operator<<(datastream& }; template -datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, as_type(obj.obj.account.to_uint64_t())); fc::raw::pack(ds, as_type(obj.obj.name.to_uint64_t())); history_serialize_container(ds, obj.db, as_type>(obj.obj.authorization)); - fc::raw::pack(ds, as_type(obj.obj.data)); + fc::raw::pack(ds, as_type(obj.obj.data)); return ds; } template -datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(obj.obj.receiver.to_uint64_t())); fc::raw::pack(ds, as_type(obj.obj.act_digest)); fc::raw::pack(ds, as_type(obj.obj.global_sequence)); fc::raw::pack(ds, as_type(obj.obj.recv_sequence)); - history_serialize_container(ds, obj.db, as_type>(obj.obj.auth_sequence)); + history_serialize_container(ds, obj.db, as_type>(obj.obj.auth_sequence)); fc::raw::pack(ds, as_type(obj.obj.code_sequence)); fc::raw::pack(ds, as_type(obj.obj.abi_sequence)); return ds; } template -datastream& operator<<(datastream& ds, const history_serial_wrapper& obj) { +ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, as_type(obj.obj.account.to_uint64_t())); fc::raw::pack(ds, as_type(obj.obj.delta)); return ds; } -inline fc::optional cap_error_code( const fc::optional& error_code ) { - fc::optional result; +inline std::optional cap_error_code(const std::optional& error_code) { + std::optional result; - if (!error_code) return result; + if (!error_code) + return result; const uint64_t upper_limit = static_cast(eosio::chain::system_error_code::generic_system_error); @@ -566,9 +600,9 @@ inline fc::optional cap_error_code( const fc::optional& erro } template -datastream& operator<<(datastream& ds, const history_context_wrapper& obj) { - bool debug_mode = obj.context; - fc::raw::pack(ds, fc::unsigned_int(0)); +ST& operator<<(ST& ds, const history_context_wrapper& obj) { + bool debug_mode = obj.context; + fc::raw::pack(ds, fc::unsigned_int(1)); fc::raw::pack(ds, as_type(obj.obj.action_ordinal)); fc::raw::pack(ds, as_type(obj.obj.creator_action_ordinal)); fc::raw::pack(ds, bool(obj.obj.receipt)); @@ -584,27 +618,28 @@ datastream& operator<<(datastream& ds, const history_context_wrapper>(obj.obj.account_ram_deltas)); + history_serialize_container(ds, obj.db, as_type>(obj.obj.account_disk_deltas)); - fc::optional e; + std::optional e; if (obj.obj.except) { if (debug_mode) e = obj.obj.except->to_string(); else e = "Y"; } - fc::raw::pack(ds, as_type>(e)); - fc::raw::pack(ds, as_type>(debug_mode ? obj.obj.error_code - : cap_error_code(obj.obj.error_code))); - + fc::raw::pack(ds, as_type>(e)); + fc::raw::pack(ds, + as_type>(debug_mode ? obj.obj.error_code : cap_error_code(obj.obj.error_code))); + fc::raw::pack(ds, as_type(obj.obj.return_value)); return ds; } template -datastream& operator<<(datastream& ds, - const history_context_wrapper, - eosio::augmented_transaction_trace>& obj) { - auto& trace = *obj.obj.trace; - bool debug_mode = obj.context.second; +ST& operator<<( + ST& ds, + const history_context_wrapper& obj) { + auto& trace = *obj.obj.trace; + bool debug_mode = obj.context.debug_mode; fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(trace.id)); if (trace.receipt) { @@ -615,13 +650,14 @@ datastream& operator<<(datastream& fc::raw::pack(ds, as_type(trace.receipt->cpu_usage_us)); fc::raw::pack(ds, as_type(trace.receipt->net_usage_words)); } else { - fc::raw::pack(ds, uint8_t(obj.context.first)); + fc::raw::pack(ds, uint8_t(obj.context.failed_status)); fc::raw::pack(ds, uint32_t(0)); fc::raw::pack(ds, fc::unsigned_int(0)); } fc::raw::pack(ds, as_type(debug_mode ? trace.elapsed.count() : 0)); fc::raw::pack(ds, as_type(trace.net_usage)); fc::raw::pack(ds, as_type(trace.scheduled)); + history_context_serialize_container(ds, obj.db, debug_mode, as_type>(trace.action_traces)); @@ -631,64 +667,109 @@ datastream& operator<<(datastream& ds, make_history_serial_wrapper(obj.db, as_type(*trace.account_ram_delta))); } - fc::optional e; + std::optional e; if (trace.except) { if (debug_mode) e = trace.except->to_string(); else e = "Y"; } - fc::raw::pack(ds, as_type>(e)); - fc::raw::pack(ds, as_type>(debug_mode ? trace.error_code - : cap_error_code(trace.error_code))); + fc::raw::pack(ds, as_type>(e)); + fc::raw::pack(ds, as_type>(debug_mode ? trace.error_code : cap_error_code(trace.error_code))); fc::raw::pack(ds, bool(trace.failed_dtrx_trace)); if (trace.failed_dtrx_trace) { - uint8_t stat = eosio::chain::transaction_receipt_header::hard_fail; + trace_receipt_context context = obj.context; + context.failed_status = eosio::chain::transaction_receipt_header::hard_fail; if (trace.receipt && trace.receipt->status.value == eosio::chain::transaction_receipt_header::soft_fail) - stat = eosio::chain::transaction_receipt_header::soft_fail; - std::pair context = std::make_pair(stat, debug_mode); + context.failed_status = eosio::chain::transaction_receipt_header::soft_fail; fc::raw::pack( // ds, make_history_context_wrapper( - obj.db, context, eosio::augmented_transaction_trace{trace.failed_dtrx_trace, obj.obj.partial})); + obj.db, context, + eosio::state_history::augmented_transaction_trace{trace.failed_dtrx_trace, obj.obj.packed_trx})); } - bool include_partial = obj.obj.partial && !trace.failed_dtrx_trace; - fc::raw::pack(ds, include_partial); - if (include_partial) { - auto& partial = *obj.obj.partial; - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(partial.expiration)); - fc::raw::pack(ds, as_type(partial.ref_block_num)); - fc::raw::pack(ds, as_type(partial.ref_block_prefix)); - fc::raw::pack(ds, as_type(partial.max_net_usage_words)); - fc::raw::pack(ds, as_type(partial.max_cpu_usage_ms)); - fc::raw::pack(ds, as_type(partial.delay_sec)); - fc::raw::pack(ds, as_type(partial.transaction_extensions)); - fc::raw::pack(ds, as_type>(partial.signatures)); - fc::raw::pack(ds, as_type>(partial.context_free_data)); + bool include_packed_trx = obj.obj.packed_trx && !trace.failed_dtrx_trace; + fc::raw::pack(ds, include_packed_trx); + if (include_packed_trx) { + const auto& pt = *obj.obj.packed_trx; + const auto& trx = pt.get_transaction(); + fc::raw::pack(ds, fc::unsigned_int( fc::get_index() )); + fc::raw::pack(ds, as_type(trx.expiration)); + fc::raw::pack(ds, as_type(trx.ref_block_num)); + fc::raw::pack(ds, as_type(trx.ref_block_prefix)); + fc::raw::pack(ds, as_type(trx.max_net_usage_words)); + fc::raw::pack(ds, as_type(trx.max_cpu_usage_ms)); + fc::raw::pack(ds, as_type(trx.delay_sec)); + fc::raw::pack(ds, as_type(trx.transaction_extensions)); + fc::raw::pack(ds, as_type>({})); } return ds; } template -datastream& operator<<(datastream& ds, - const history_context_wrapper& obj) { - std::pair context = std::make_pair(eosio::chain::transaction_receipt_header::hard_fail, obj.context); - ds << make_history_context_wrapper(obj.db, context, obj.obj); +ST& operator<<(ST& ds, const eosio::state_history::get_blocks_result_v0& obj) { + fc::raw::pack(ds, obj.head); + fc::raw::pack(ds, obj.last_irreversible); + fc::raw::pack(ds, obj.this_block); + fc::raw::pack(ds, obj.prev_block); + eosio::state_history::pack_big_bytes(ds, obj.block); + eosio::state_history::pack_big_bytes(ds, obj.traces); + eosio::state_history::pack_big_bytes(ds, obj.deltas); + return ds; +} + +template +void pack_for_blocks_result_v1(ST& ds, const eosio::state_history::signed_block_ptr_variant& obj) { + uint8_t which = obj.index(); + std::visit([&ds, which](const auto& ptr) { + fc::raw::pack(ds, bool(ptr)); + if (ptr) { + fc::raw::pack(ds, which); + fc::raw::pack(ds, *ptr); + } + }, obj); +} + +template +void pack_for_blocks_result_v2(ST& ds, const eosio::state_history::signed_block_ptr_variant& obj) { + uint8_t which = obj.index(); + std::visit([&ds, which](const auto& ptr) { + if (ptr) { + fc::datastream> strm; + fc::raw::pack(strm, which); + fc::raw::pack(strm, *ptr); + fc::raw::pack(ds, strm.storage()); + } else { + fc::raw::pack(ds, unsigned_int(0)); + } + }, obj); +} + +template +ST& operator<<(ST& ds, const eosio::state_history::get_blocks_result_v1& obj) { + fc::raw::pack(ds, obj.head); + fc::raw::pack(ds, obj.last_irreversible); + fc::raw::pack(ds, obj.this_block); + fc::raw::pack(ds, obj.prev_block); + pack_for_blocks_result_v1(ds, obj.block); + fc::raw::pack(ds, obj.traces); + fc::raw::pack(ds, obj.deltas); return ds; } + template -datastream& operator<<(datastream& ds, const eosio::get_blocks_result_v0& obj) { +ST& operator<<(ST& ds, const eosio::state_history::get_blocks_result_v2& obj) { fc::raw::pack(ds, obj.head); fc::raw::pack(ds, obj.last_irreversible); fc::raw::pack(ds, obj.this_block); fc::raw::pack(ds, obj.prev_block); - history_pack_big_bytes(ds, obj.block); - history_pack_big_bytes(ds, obj.traces); - history_pack_big_bytes(ds, obj.deltas); + pack_for_blocks_result_v2(ds, obj.block); + fc::raw::pack(ds, obj.block_header); + fc::raw::pack(ds, obj.traces); + fc::raw::pack(ds, obj.deltas); return ds; } diff --git a/libraries/state_history/include/eosio/state_history/trace_converter.hpp b/libraries/state_history/include/eosio/state_history/trace_converter.hpp new file mode 100644 index 00000000000..eb9ad9c742c --- /dev/null +++ b/libraries/state_history/include/eosio/state_history/trace_converter.hpp @@ -0,0 +1,237 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace eosio { +namespace state_history { +namespace trace_converter { + +using eosio::chain::overloaded; +using eosio::chain::packed_transaction; +using eosio::chain::state_history_exception; +using eosio::chain::transaction_id_type; +using prunable_data_type = packed_transaction::prunable_data_type; + +struct ondisk_prunable_full_legacy { + std::vector signatures; + std::vector context_free_segments; + eosio::chain::digest_type digest; + eosio::chain::digest_type prunable_digest() const { return digest; } +}; + +using ondisk_prunable_data_t = std::variant; + + +inline prunable_data_type to_prunable_data_type(ondisk_prunable_data_t&& data) { + return std::visit( + overloaded{[](ondisk_prunable_full_legacy&& v) { + prunable_data_type::full y{std::move(v.signatures), std::move(v.context_free_segments)}; + return prunable_data_type{y}; + }, + [](auto&& v) { return prunable_data_type{std::move(v)}; }}, std::move(data)); +} + +template +void for_each_packed_transaction(const eosio::state_history::augmented_transaction_trace& obj, const Lambda& lambda) { + auto& trace = *obj.trace; + if (trace.failed_dtrx_trace) { + for_each_packed_transaction( + eosio::state_history::augmented_transaction_trace{trace.failed_dtrx_trace, obj.packed_trx}, lambda); + } + bool include_packed_trx = obj.packed_trx && !trace.failed_dtrx_trace; + if (include_packed_trx) { + lambda(*obj.packed_trx); + } +} + +/// used to traverse every packed_transaction inside traces before the pruned_data has been serialized +template +void for_each_packed_transaction(const std::vector& traces, + const Lambda& lambda) { + for (const auto& v : traces) { + for_each_packed_transaction(v, lambda); + } +} + +template +void pack(OSTREAM&& strm, const T& obj, compression_type compression) { + switch (compression) { + case compression_type::zlib: + zlib_pack(strm, obj); + break; + case compression_type::none: + fc::raw::pack(strm, obj); + break; + default: + EOS_ASSERT(false, state_history_exception, "unsupported compression type"); + } +} + +template +void unpack(ISTREAM&& strm, T& obj, compression_type compression) { + switch (compression) { + case compression_type::zlib: + zlib_unpack(strm, obj); + break; + case compression_type::none: + fc::raw::unpack(strm, obj); + break; + default: + EOS_ASSERT(false, state_history_exception, "unsupported compression type"); + } +} + +template +size_t pack(OSTREAM&& strm, const prunable_data_type& obj, compression_type compression) { + auto start_pos = strm.tellp(); + fc::raw::pack(strm, static_cast(obj.prunable_data.index())); + + std::visit(overloaded{[&strm](const prunable_data_type::none& data) { fc::raw::pack(strm, data); }, + [&strm, compression](const prunable_data_type::full_legacy& data) { + fc::raw::pack(strm, data.signatures); + pack(strm, data.context_free_segments, compression); + fc::raw::pack(strm, data.prunable_digest()); + }, + [&strm, compression](const auto& data) { + fc::raw::pack(strm, data.signatures); + pack(strm, data.context_free_segments, compression); + }}, obj.prunable_data); + auto consumed = strm.tellp() - start_pos; + return std::max(sizeof(uint8_t) + sizeof(chain::digest_type), consumed); +} + +template +void unpack(ISTREAM&& strm, ondisk_prunable_data_t& prunable, compression_type compression) { + uint8_t tag; + fc::raw::unpack(strm, tag); + // set_variant_by_index(prunable, tag); + fc::from_index(prunable, tag); + std::visit(overloaded{[&strm](prunable_data_type::none& data) { fc::raw::unpack(strm, data); }, + [&strm, compression](ondisk_prunable_full_legacy& data) { + fc::raw::unpack(strm, data.signatures); + unpack(strm, data.context_free_segments, compression); + fc::raw::unpack(strm, data.digest); + }, + [&strm, compression](auto& data) { + fc::raw::unpack(strm, data.signatures); + unpack(strm, data.context_free_segments, compression); + }}, prunable); +} + +/// used to traverse each trace along with its associated unpacked prunable_data +template +void visit_deserialized_trace(ISTREAM& strm, transaction_trace& trace, compression_type compression, + Visitor&& visitor) { + auto& trace_v0 = std::get(trace); + if (trace_v0.failed_dtrx_trace.size()) { + // failed_dtrx_trace have at most one element because it is encoded as an optional + visit_deserialized_trace(strm, trace_v0.failed_dtrx_trace[0].recurse, compression, + std::forward(visitor)); + } + if (trace_v0.partial) { + ondisk_prunable_data_t prunable; + unpack(strm, prunable, compression); + visitor(trace_v0, std::move(prunable)); + } +} + +template +struct trace_pruner { + bytes& buffer; // the buffer contains the entire prunable section, which is the storage used by read_strm + uint64_t last_read_pos = 0; + std::vector& ids; /// the transaction ids to be pruned + fc::datastream read_strm; + OSTREAM& write_strm; + uint64_t change_position = 0; /// when it's nonzero, represents the offset to data where the content has been changed + compression_type compression; + + trace_pruner(bytes& entry_payload, std::vector& ids, OSTREAM& ostrm) + : buffer(entry_payload) + , ids(ids) + , read_strm(entry_payload.data(), entry_payload.size()) + , write_strm(ostrm) { + fc::raw::unpack(read_strm, compression); + } + + void operator()(transaction_trace_v0& trace, ondisk_prunable_data_t&& incoming_prunable) { + auto itr = std::find(ids.begin(), ids.end(), trace.id); + if (itr != ids.end()) { + // the incoming trace matches the one of ids to be pruned + if (change_position == 0) + change_position = write_strm.tellp(); + auto digest = std::visit([](const auto& v) { return v.prunable_digest(); }, incoming_prunable); + uint8_t which = fc::get_index(); + fc::raw::pack(write_strm, std::make_pair(which, digest)); + ids.erase(itr); + } else if (change_position == 0) { + // no change so far, skip the original serialized prunable_data bytes + write_strm.skip(read_strm.tellp() - last_read_pos); + } else { + // change detected, shift the original serialized prunable_data bytes to new location + write_strm.write(buffer.data() + last_read_pos, read_strm.tellp() - last_read_pos); + } + last_read_pos = read_strm.tellp(); + } +}; + +template +void pack(OSTREAM&& strm, const chainbase::database& db, bool trace_debug_mode, + const std::vector& traces, compression_type compression) { + + // In version 1 of SHiP traces log disk format, it log entry consists of 3 parts. + // 1. a zlib compressed unprunable section contains the serialization of the vector of traces excluding + // the prunable_data data (i.e. signatures and context free data) + // 2. an uint8_t tag indicating the compression mechanism for the context free data inside the prunable section. + // 3. a prunable section contains the serialization of the vector of ondisk_prunable_data_t. + zlib_pack(strm, make_history_context_wrapper(db, trace_receipt_context{.debug_mode = trace_debug_mode}, traces)); + fc::raw::pack(strm, static_cast(compression)); + const auto pos = strm.tellp(); + size_t size_with_padding = 0; + for_each_packed_transaction(traces, [&strm, &size_with_padding, compression](const chain::packed_transaction& pt) { + size_with_padding += trace_converter::pack(strm, pt.get_prunable_data(), compression); + }); + strm.seekp(pos + size_with_padding); +} + +template +void unpack(ISTREAM&& strm, std::vector& traces) { + zlib_unpack(strm, traces); + uint8_t compression; + fc::raw::unpack(strm, compression); + for (auto& trace : traces) { + visit_deserialized_trace(strm, trace, static_cast(compression), + [](transaction_trace_v0& trace, ondisk_prunable_data_t&& prunable_data) { + auto& ptrx = std::get(*trace.partial); + ptrx.prunable_data = to_prunable_data_type(std::move(prunable_data)); + }); + } +} + +template +void prune_traces(IOSTREAM&& strm, uint32_t entry_len, std::vector& ids) { + std::vector traces; + size_t unprunable_section_pos = strm.tellp(); + zlib_unpack(strm, traces); + size_t prunable_section_pos = strm.tellp(); + std::vector buffer(unprunable_section_pos + entry_len - prunable_section_pos); + strm.read(buffer.data(), buffer.size()); + // restore the stream position to the start of prunable section so it can used for writing + strm.seekp(prunable_section_pos); + + auto pruner = trace_pruner{buffer, ids, strm}; + for (auto& trace : traces) { + visit_deserialized_trace(pruner.read_strm, trace, pruner.compression, pruner); + } +} + +} // namespace trace_converter +} // namespace state_history +} // namespace eosio + +FC_REFLECT(eosio::state_history::trace_converter::ondisk_prunable_full_legacy, + (signatures)(context_free_segments)(digest)) diff --git a/libraries/state_history/include/eosio/state_history/transaction_trace_cache.hpp b/libraries/state_history/include/eosio/state_history/transaction_trace_cache.hpp new file mode 100644 index 00000000000..cd47a15fe3c --- /dev/null +++ b/libraries/state_history/include/eosio/state_history/transaction_trace_cache.hpp @@ -0,0 +1,25 @@ +#pragma once + +#include +#include + +namespace eosio { +namespace state_history { + +using chain::block_state_ptr; +using chain::packed_transaction_ptr; +using chain::transaction_id_type; + +struct transaction_trace_cache { + std::map cached_traces; + std::optional onblock_trace; + + void add_transaction(const transaction_trace_ptr& trace, const packed_transaction_ptr& transaction); + + std::vector prepare_traces(const block_state_ptr& block_state); + + void clear(); +}; + +} // namespace state_history +} // namespace eosio diff --git a/libraries/state_history/include/eosio/state_history/types.hpp b/libraries/state_history/include/eosio/state_history/types.hpp new file mode 100644 index 00000000000..49a7b616b75 --- /dev/null +++ b/libraries/state_history/include/eosio/state_history/types.hpp @@ -0,0 +1,341 @@ +#pragma once + +#include +#include + +namespace eosio { +namespace state_history { + +using chain::block_id_type; +using chain::bytes; +using chain::extensions_type; +using chain::signature_type; +using chain::signed_transaction; +using chain::transaction_trace_ptr; + +using compression_type = chain::packed_transaction::prunable_data_type::compression_type; + +template +struct big_vector_wrapper { + T obj; +}; + +template +inline void pack_varuint64(ST& ds, uint64_t val) { + do { + uint8_t b = uint8_t(val) & 0x7f; + val >>= 7; + b |= ((val > 0) << 7); + ds.write((char*)&b, 1); + } while (val); +} + +template +void pack_big_bytes(ST& ds, const eosio::chain::bytes& v) { + pack_varuint64(ds, v.size()); + if (v.size()) + ds.write(&v.front(), v.size()); +} + +template +void pack_big_bytes(ST& ds, const std::optional& v) { + fc::raw::pack(ds, v.has_value()); + if (v) + pack_big_bytes(ds, *v); +} + +template +class opaque { + std::vector data; + + public: + opaque() = default; + explicit opaque(std::vector&& serialized) + : data(std::move(serialized)) {} + opaque(const T& value) { data = fc::raw::pack(value); } + + opaque& operator=(const T& value) { + data = fc::raw::pack(value); + return *this; + } + opaque& operator=(std::vector&& serialized) { + data = std::move(serialized); + return *this; + } + + template + void pack_to(ST& ds) const { + // we need to pack as big vector because it can be used to hold the state delta object + // which would be as large as the eos snapshot when the nodeos restarted from a snapshot. + pack_big_bytes(ds, this->data); + } + + bool has_value() const { return data.size(); } +}; + +struct augmented_transaction_trace { + transaction_trace_ptr trace; + chain::packed_transaction_ptr packed_trx; + + augmented_transaction_trace() = default; + augmented_transaction_trace(const augmented_transaction_trace&) = default; + augmented_transaction_trace(augmented_transaction_trace&&) = default; + + augmented_transaction_trace(const transaction_trace_ptr& trace) + : trace{trace} {} + + augmented_transaction_trace(const transaction_trace_ptr& trace, const chain::packed_transaction_ptr& packed_trx) + : trace{trace} + , packed_trx{packed_trx} {} + + augmented_transaction_trace& operator=(const augmented_transaction_trace&) = default; + augmented_transaction_trace& operator=(augmented_transaction_trace&&) = default; +}; + +struct table_delta { + fc::unsigned_int struct_version = 1; + std::string name{}; + state_history::big_vector_wrapper>> rows{}; +}; + +struct block_position { + uint32_t block_num = 0; + block_id_type block_id = {}; +}; + +struct get_status_request_v0 {}; + +struct get_status_result_v0 { + block_position head = {}; + block_position last_irreversible = {}; + uint32_t trace_begin_block = 0; + uint32_t trace_end_block = 0; + uint32_t chain_state_begin_block = 0; + uint32_t chain_state_end_block = 0; + fc::sha256 chain_id = {}; +}; + +struct get_blocks_result_v1; +struct get_blocks_result_v2; +struct get_blocks_request_v0 { + uint32_t start_block_num = 0; + uint32_t end_block_num = 0; + uint32_t max_messages_in_flight = 0; + std::vector have_positions = {}; + bool irreversible_only = false; + bool fetch_block = false; + bool fetch_traces = false; + bool fetch_deltas = false; + using response_type = get_blocks_result_v1; +}; + +struct get_blocks_request_v1 : get_blocks_request_v0 { + bool fetch_block_header = false; + + get_blocks_request_v1() = default; + get_blocks_request_v1(const get_blocks_request_v0& v0) + : get_blocks_request_v0(v0) + , fetch_block_header(false) {} + using response_type = get_blocks_result_v2; +}; + +struct get_blocks_ack_request_v0 { + uint32_t num_messages = 0; +}; + +struct get_blocks_result_v0 { + block_position head; + block_position last_irreversible; + std::optional this_block; + std::optional prev_block; + std::optional block; + std::optional traces; + std::optional deltas; +}; + +using state_request = std::variant; + +struct account_auth_sequence { + uint64_t account = {}; + uint64_t sequence = {}; +}; + +struct account_delta { + uint64_t account = {}; + uint64_t delta = {}; +}; + +struct permission_level { + uint64_t actor = {}; + uint64_t permission = {}; +}; + +struct action_receipt_v0 { + uint64_t receiver = {}; + eosio::chain::digest_type act_digest = {}; + uint64_t global_sequence = {}; + uint64_t recv_sequence = {}; + std::vector auth_sequence = {}; + fc::unsigned_int code_sequence = {}; + fc::unsigned_int abi_sequence = {}; +}; + +using action_receipt = std::variant; + +struct action { + uint64_t account = {}; + uint64_t name = {}; + std::vector authorization = {}; + bytes data = {}; +}; + +struct action_trace_v0 { + fc::unsigned_int action_ordinal = {}; + fc::unsigned_int creator_action_ordinal = {}; + std::optional receipt = {}; + uint64_t receiver = {}; + action act = {}; + bool context_free = {}; + int64_t elapsed = {}; + std::string console = {}; + std::vector account_ram_deltas = {}; + std::optional except = {}; + std::optional error_code = {}; +}; + +struct action_trace_v1 { + fc::unsigned_int action_ordinal = {}; + fc::unsigned_int creator_action_ordinal = {}; + std::optional receipt = {}; + uint64_t receiver = {}; + action act = {}; + bool context_free = {}; + int64_t elapsed = {}; + std::string console = {}; + std::vector account_ram_deltas = {}; + std::vector account_disk_deltas = {}; + std::optional except = {}; + std::optional error_code = {}; + bytes return_value = {}; +}; + +using action_trace = std::variant; + +struct partial_transaction_v0 { + eosio::chain::time_point_sec expiration = {}; + uint16_t ref_block_num = {}; + uint32_t ref_block_prefix = {}; + fc::unsigned_int max_net_usage_words = {}; + uint8_t max_cpu_usage_ms = {}; + fc::unsigned_int delay_sec = {}; + eosio::chain::extensions_type transaction_extensions = {}; + std::vector signatures = {}; + std::vector context_free_data = {}; +}; + +using prunable_data_type = eosio::chain::packed_transaction::prunable_data_type; +struct partial_transaction_v1 { + eosio::chain::time_point_sec expiration = {}; + uint16_t ref_block_num = {}; + uint32_t ref_block_prefix = {}; + fc::unsigned_int max_net_usage_words = {}; + uint8_t max_cpu_usage_ms = {}; + fc::unsigned_int delay_sec = {}; + eosio::chain::extensions_type transaction_extensions = {}; + std::optional prunable_data = {}; +}; + +using partial_transaction = std::variant; + +struct transaction_trace_recurse; + +struct transaction_trace_v0 { + using transaction_trace = std::variant; + eosio::chain::digest_type id = {}; + uint8_t status = {}; + uint32_t cpu_usage_us = {}; + fc::unsigned_int net_usage_words = {}; + int64_t elapsed = {}; + uint64_t net_usage = {}; + bool scheduled = {}; + std::vector action_traces = {}; + std::optional account_ram_delta = {}; + std::optional except = {}; + std::optional error_code = {}; + + // semantically, this should be std::optional; + // it is represented as vector because optional cannot be used for incomplete type + std::vector failed_dtrx_trace = {}; + std::optional partial = {}; +}; + +using transaction_trace = std::variant; +struct transaction_trace_recurse { + transaction_trace recurse; +}; + +using signed_block_ptr_variant = std::variant; + +struct get_blocks_result_v1 { + block_position head; + block_position last_irreversible; + std::optional this_block; + std::optional prev_block; + signed_block_ptr_variant block; // packed as std::optional> + opaque> traces; + opaque> deltas; + + bool has_value() const { + return std::visit([](auto b) -> bool { return b.get(); }, block) || traces.has_value() || deltas.has_value(); + } +}; + +struct get_blocks_result_v2 { + block_position head; + block_position last_irreversible; + std::optional this_block; + std::optional prev_block; + signed_block_ptr_variant block; // packed as opaque> + opaque block_header; + opaque> traces; + opaque> deltas; + bool has_value() const { + return std::visit([](auto b) -> bool { return b.get(); }, block) || traces.has_value() || deltas.has_value() || block_header.has_value(); + } +}; + + +using state_result = std::variant; + +} // namespace state_history +} // namespace eosio + +// clang-format off +FC_REFLECT(eosio::state_history::table_delta, (struct_version)(name)(rows)); +FC_REFLECT(eosio::state_history::block_position, (block_num)(block_id)); +FC_REFLECT_EMPTY(eosio::state_history::get_status_request_v0); +FC_REFLECT(eosio::state_history::get_status_result_v0, (head)(last_irreversible)(trace_begin_block)(trace_end_block)(chain_state_begin_block)(chain_state_end_block)(chain_id)); +FC_REFLECT(eosio::state_history::get_blocks_request_v0, (start_block_num)(end_block_num)(max_messages_in_flight)(have_positions)(irreversible_only)(fetch_block)(fetch_traces)(fetch_deltas)); +FC_REFLECT_DERIVED(eosio::state_history::get_blocks_request_v1, (eosio::state_history::get_blocks_request_v0), (fetch_block_header)); +FC_REFLECT(eosio::state_history::get_blocks_ack_request_v0, (num_messages)); + +FC_REFLECT(eosio::state_history::account_auth_sequence, (account)(sequence)); +FC_REFLECT(eosio::state_history::account_delta, (account)(delta)); +FC_REFLECT(eosio::state_history::permission_level,(actor)(permission)); +FC_REFLECT(eosio::state_history::action_receipt_v0, (receiver)(act_digest)(global_sequence)(recv_sequence)(auth_sequence)(code_sequence)(abi_sequence)); +FC_REFLECT(eosio::state_history::action, (account)(name)(authorization)(data)); +FC_REFLECT(eosio::state_history::action_trace_v0, (action_ordinal)(creator_action_ordinal)(receipt)(receiver)(act)(context_free)(elapsed)(console)(account_ram_deltas)(except)(error_code)); +FC_REFLECT(eosio::state_history::action_trace_v1, (action_ordinal)(creator_action_ordinal)(receipt)(receiver)(act)(context_free)(elapsed)(console)(account_ram_deltas)(account_disk_deltas)(except)(error_code)(return_value)); +FC_REFLECT(eosio::state_history::partial_transaction_v0, (expiration)(ref_block_num)(ref_block_prefix)(max_net_usage_words)(max_cpu_usage_ms)(delay_sec)(transaction_extensions)(signatures)(context_free_data)); +FC_REFLECT(eosio::state_history::partial_transaction_v1, (expiration)(ref_block_num)(ref_block_prefix)(max_net_usage_words)(max_cpu_usage_ms)(delay_sec)(transaction_extensions)(prunable_data)); +FC_REFLECT(eosio::state_history::transaction_trace_v0,(id)(status)(cpu_usage_us)(net_usage_words)(elapsed)(net_usage)(scheduled)(action_traces)(account_ram_delta)(except)(error_code)(failed_dtrx_trace)(partial)); +FC_REFLECT(eosio::state_history::transaction_trace_recurse, (recurse)); +// clang-format on + +namespace fc { +template +ST& operator<<(ST& strm, const eosio::state_history::opaque& v) { + v.pack_to(strm); + return strm; +} +} // namespace fc diff --git a/libraries/state_history/log.cpp b/libraries/state_history/log.cpp new file mode 100644 index 00000000000..790dcd725dc --- /dev/null +++ b/libraries/state_history/log.cpp @@ -0,0 +1,407 @@ +#include +#include +#include +#include +#include + +namespace eosio { + +uint64_t state_history_log_data::payload_size_at(uint64_t pos) const { + EOS_ASSERT(file.size() >= pos + sizeof(state_history_log_header), chain::state_history_exception, + "corrupt ${name}: invalid entry size at at position ${pos}", ("name", filename)("pos", pos)); + + fc::datastream ds(file.const_data() + pos, sizeof(state_history_log_header)); + state_history_log_header header; + fc::raw::unpack(ds, header); + + EOS_ASSERT(is_ship(header.magic) && is_ship_supported_version(header.magic), chain::state_history_exception, + "corrupt ${name}: invalid header for entry at position ${pos}", ("name", filename)("pos", pos)); + + EOS_ASSERT(file.size() >= pos + sizeof(state_history_log_header) + header.payload_size, + chain::state_history_exception, "corrupt ${name}: invalid payload size for entry at position ${pos}", + ("name", filename)("pos", pos)); + return header.payload_size; +} + +void state_history_log_data::construct_index(const fc::path& index_file_name) const { + fc::cfile index; + index.set_file_path(index_file_name); + index.open("w+b"); + + uint64_t pos = 0; + while (pos < file.size()) { + uint64_t payload_size = payload_size_at(pos); + index.write(reinterpret_cast(&pos), sizeof(pos)); + pos += (sizeof(state_history_log_header) + payload_size + sizeof(uint64_t)); + } +} + +state_history_log::state_history_log(const char* const name, const state_history_config& config) + : name(name) { + catalog.open(config.log_dir, config.retained_dir, config.archive_dir, name); + catalog.max_retained_files = config.max_retained_files; + this->stride = config.stride; + open_log(config.log_dir / (std::string(name) + ".log")); + open_index(config.log_dir / (std::string(name) + ".index")); +} + +void state_history_log::read_header(state_history_log_header& header, bool assert_version) { + char bytes[state_history_log_header_serial_size]; + read_log.read(bytes, sizeof(bytes)); + fc::datastream ds(bytes, sizeof(bytes)); + fc::raw::unpack(ds, header); + EOS_ASSERT(!ds.remaining(), chain::state_history_exception, "state_history_log_header_serial_size mismatch"); + version = get_ship_version(header.magic); + if (assert_version) + EOS_ASSERT(is_ship(header.magic) && is_ship_supported_version(header.magic), chain::state_history_exception, + "corrupt ${name}.log (0)", ("name", name)); +} + +void state_history_log::write_header(const state_history_log_header& header) { fc::raw::pack(write_log, header); } + +// returns cfile positioned at payload +void state_history_log::get_entry_header(state_history_log::block_num_type block_num, + state_history_log_header& header) { + EOS_ASSERT(block_num >= _begin_block && block_num < _end_block, chain::state_history_exception, + "read non-existing block in ${name}.log", ("name", name)); + read_log.seek(get_pos(block_num)); + read_header(header); +} + +std::optional state_history_log::get_block_id(state_history_log::block_num_type block_num) { + auto result = catalog.id_for_block(block_num); + if (!result && block_num >= _begin_block && block_num < _end_block) { + state_history_log_header header; + get_entry_header(block_num, header); + return header.block_id; + } + return result; +} + +bool state_history_log::get_last_block(uint64_t size) { + state_history_log_header header; + uint64_t suffix; + read_log.seek(size - sizeof(suffix)); + read_log.read((char*)&suffix, sizeof(suffix)); + if (suffix > size || suffix + state_history_log_header_serial_size > size) { + elog("corrupt ${name}.log (2)", ("name", name)); + return false; + } + read_log.seek(suffix); + read_header(header, false); + if (!is_ship(header.magic) || !is_ship_supported_version(header.magic) || + suffix + state_history_log_header_serial_size + header.payload_size + sizeof(suffix) != size) { + elog("corrupt ${name}.log (3)", ("name", name)); + return false; + } + _end_block = chain::block_header::num_from_id(header.block_id) + 1; + last_block_id = header.block_id; + if (_begin_block >= _end_block) { + elog("corrupt ${name}.log (4)", ("name", name)); + return false; + } + return true; +} + +void state_history_log::recover_blocks(uint64_t size) { + ilog("recover ${name}.log", ("name", name)); + uint64_t pos = 0; + uint32_t num_found = 0; + while (true) { + state_history_log_header header; + if (pos + state_history_log_header_serial_size > size) + break; + read_log.seek(pos); + read_header(header, false); + uint64_t suffix; + if (!is_ship(header.magic) || !is_ship_supported_version(header.magic) || header.payload_size > size || + pos + state_history_log_header_serial_size + header.payload_size + sizeof(suffix) > size) { + EOS_ASSERT(!is_ship(header.magic) || is_ship_supported_version(header.magic), chain::state_history_exception, + "${name}.log has an unsupported version", ("name", name)); + break; + } + read_log.seek(pos + state_history_log_header_serial_size + header.payload_size); + read_log.read((char*)&suffix, sizeof(suffix)); + if (suffix != pos) + break; + pos = pos + state_history_log_header_serial_size + header.payload_size + sizeof(suffix); + if (!(++num_found % 10000)) { + dlog("${num_found} blocks found, log pos = ${pos}", ("num_found", num_found)("pos", pos)); + } + } + read_log.flush(); + boost::filesystem::resize_file(read_log.get_file_path(), pos); + read_log.flush(); + EOS_ASSERT(get_last_block(pos), chain::state_history_exception, "recover ${name}.log failed", ("name", name)); +} + +void state_history_log::open_log(bfs::path log_filename) { + write_log.set_file_path(log_filename); + read_log.set_file_path(log_filename); + write_log.open("a+b"); // create file if not exists + write_log.close(); + write_log.open("rb+"); // fseek doesn't work for append mode, need to open with update mode. + write_log.seek_end(0); + read_log.open("rb"); + uint64_t size = write_log.tellp(); + if (size >= state_history_log_header_serial_size) { + state_history_log_header header; + read_log.seek(0); + read_header(header, false); + EOS_ASSERT(is_ship(header.magic) && is_ship_supported_version(header.magic) && + state_history_log_header_serial_size + header.payload_size + sizeof(uint64_t) <= size, + chain::state_history_exception, "corrupt ${name}.log (1)", ("name", name)); + _begin_block = chain::block_header::num_from_id(header.block_id); + last_block_id = header.block_id; + if (!get_last_block(size)) + recover_blocks(size); + ilog("${name}.log has blocks ${b}-${e}", ("name", name)("b", _begin_block)("e", _end_block - 1)); + } else { + EOS_ASSERT(!size, chain::state_history_exception, "corrupt ${name}.log (5)", ("name", name)); + ilog("${name}.log is empty", ("name", name)); + } +} + +void state_history_log::open_index(bfs::path index_filename) { + index.set_file_path(index_filename); + index.open("a+b"); // std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app + index.seek_end(0); + if (index.tellp() == (static_cast(_end_block) - _begin_block) * sizeof(uint64_t)) + return; + ilog("Regenerate ${name}.index", ("name", name)); + index.close(); + + state_history_log_data(read_log.get_file_path()).construct_index(index_filename); + index.open("a+b"); // std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app + index.seek_end(0); +} + +state_history_log::file_position_type state_history_log::get_pos(state_history_log::block_num_type block_num) { + uint64_t pos; + index.seek((block_num - _begin_block) * sizeof(pos)); + index.read((char*)&pos, sizeof(pos)); + return pos; +} + +void state_history_log::truncate(state_history_log::block_num_type block_num) { + write_log.flush(); + index.flush(); + auto first_block_num = catalog.empty() ? _begin_block : catalog.first_block_num(); + auto new_begin_block_num = catalog.truncate(block_num, read_log.get_file_path()); + + if (new_begin_block_num > 0) { + // in this case, the original index/log file has been replaced from some files from the catalog, we need to + // reopen read_log, write_log and index. + index.close(); + index.open("rb"); + _begin_block = new_begin_block_num; + } + + uint64_t num_removed; + if (block_num <= _begin_block) { + num_removed = _end_block - first_block_num; + boost::filesystem::resize_file(read_log.get_file_path(), 0); + boost::filesystem::resize_file(index.get_file_path(), 0); + _begin_block = _end_block = block_num; + } else { + num_removed = _end_block - block_num; + uint64_t pos = get_pos(block_num); + boost::filesystem::resize_file(read_log.get_file_path(), pos); + boost::filesystem::resize_file(index.get_file_path(), (block_num - _begin_block) * sizeof(uint64_t)); + _end_block = block_num; + } + + read_log.close(); + read_log.open("rb"); + write_log.close(); + write_log.open("rb+"); + index.close(); + index.open("a+b"); + + ilog("fork or replay: removed ${n} blocks from ${name}.log", ("n", num_removed)("name", name)); +} + +std::pair +state_history_log::write_entry_header(const state_history_log_header& header, const chain::block_id_type& prev_id) { + block_num_type block_num = chain::block_header::num_from_id(header.block_id); + EOS_ASSERT(_begin_block == _end_block || block_num <= _end_block, chain::state_history_exception, + "missed a block in ${name}.log", ("name", name)); + + if (_begin_block != _end_block && block_num > _begin_block) { + if (block_num == _end_block) { + EOS_ASSERT(prev_id == last_block_id, chain::state_history_exception, "missed a fork change in ${name}.log", + ("name", name)); + } else { + state_history_log_header prev; + get_entry_header(block_num - 1, prev); + EOS_ASSERT(prev_id == prev.block_id, chain::state_history_exception, "missed a fork change in ${name}.log", + ("name", name)); + } + } + + if (block_num < _end_block) { + truncate(block_num); + } + write_log.seek_end(0); + file_position_type pos = write_log.tellp(); + write_header(header); + return std::make_pair(block_num, pos); +} + +void state_history_log::write_entry_position(const state_history_log_header& header, + state_history_log::file_position_type pos, + state_history_log::block_num_type block_num) { + uint64_t end = write_log.tellp(); + uint64_t payload_start_pos = pos + state_history_log_header_serial_size; + uint64_t payload_size = end - payload_start_pos; + write_log.write((char*)&pos, sizeof(pos)); + write_log.seek(payload_start_pos - sizeof(header.payload_size)); + write_log.write((char*)&payload_size, sizeof(header.payload_size)); + write_log.seek_end(0); + + index.seek_end(0); + index.write((char*)&pos, sizeof(pos)); + if (_begin_block == _end_block) + _begin_block = block_num; + _end_block = block_num + 1; + last_block_id = header.block_id; + + write_log.flush(); + index.flush(); + + if (block_num % stride == 0) { + split_log(); + } +} + +void state_history_log::split_log() { + index.close(); + read_log.close(); + write_log.close(); + + catalog.add(_begin_block, _end_block - 1, read_log.get_file_path().parent_path(), name); + + _begin_block = _end_block; + + write_log.open("w+b"); + read_log.open("rb"); + index.open("w+b"); +} + +state_history_traces_log::state_history_traces_log(const state_history_config& config) + : state_history_log("trace_history", config) {} + +chain::bytes state_history_traces_log::get_log_entry(block_num_type block_num) { + + auto get_traces_bin = [block_num](auto& ds, uint32_t version, std::size_t size) { + auto start_pos = ds.tellp(); + try { + if (version == 0) { + return state_history::zlib_decompress(ds); + } + else { + std::vector traces; + state_history::trace_converter::unpack(ds, traces); + return fc::raw::pack(traces); + } + } catch (fc::exception& ex) { + std::vector trace_data(size); + ds.seekp(start_pos); + ds.read(trace_data.data(), size); + + fc::cfile output; + char filename[PATH_MAX]; + snprintf(filename, PATH_MAX, "invalid_trace_%u_v%u.bin", block_num, version); + output.set_file_path(filename); + output.open("w"); + output.write(trace_data.data(), size); + + ex.append_log(FC_LOG_MESSAGE(error, + "trace data for block ${block_num} has been written to ${filename} for debugging", + ("block_num", block_num)("filename", filename))); + + throw ex; + } + }; + + auto [ds, version] = catalog.ro_stream_for_block(block_num); + if (ds.remaining()) { + return get_traces_bin(ds, version, ds.remaining()); + } + + if (block_num < begin_block() || block_num >= end_block()) + return {}; + state_history_log_header header; + get_entry_header(block_num, header); + return get_traces_bin(read_log, get_ship_version(header.magic), header.payload_size); +} + + +void state_history_traces_log::prune_transactions(state_history_log::block_num_type block_num, + std::vector& ids) { + auto [ds, version] = catalog.rw_stream_for_block(block_num); + + if (ds.remaining()) { + EOS_ASSERT(version > 0, chain::state_history_exception, + "The trace log version 0 does not support transaction pruning."); + state_history::trace_converter::prune_traces(ds, ds.remaining(), ids); + return; + } + + if (block_num < begin_block() || block_num >= end_block()) + return; + state_history_log_header header; + get_entry_header(block_num, header); + EOS_ASSERT(get_ship_version(header.magic) > 0, chain::state_history_exception, + "The trace log version 0 does not support transaction pruning."); + write_log.seek(read_log.tellp()); + state_history::trace_converter::prune_traces(write_log, header.payload_size, ids); + write_log.flush(); +} + +void state_history_traces_log::store(const chainbase::database& db, const chain::block_state_ptr& block_state) { + + state_history_log_header header{.magic = ship_magic(ship_current_version), .block_id = block_state->id}; + auto trace = cache.prepare_traces(block_state); + + this->write_entry(header, block_state->block->previous, [&](auto& stream) { + state_history::trace_converter::pack(stream, db, trace_debug_mode, trace, compression); + }); +} + +bool state_history_traces_log::exists(bfs::path state_history_dir) { + return bfs::exists(state_history_dir / "trace_history.log") && + bfs::exists(state_history_dir / "trace_history.index"); +} + +state_history_chain_state_log::state_history_chain_state_log(const state_history_config& config) + : state_history_log("chain_state_history", config) {} + +chain::bytes state_history_chain_state_log::get_log_entry(block_num_type block_num) { + + auto [ds, _] = catalog.ro_stream_for_block(block_num); + if (ds.remaining()) { + return state_history::zlib_decompress(ds); + } + + if (block_num < begin_block() || block_num >= end_block()) + return {}; + state_history_log_header header; + get_entry_header(block_num, header); + return state_history::zlib_decompress(read_log); +} + +void state_history_chain_state_log::store(const chain::combined_database& db, + const chain::block_state_ptr& block_state) { + bool fresh = this->begin_block() == this->end_block(); + if (fresh) + ilog("Placing initial state in block ${n}", ("n", block_state->block->block_num())); + + using namespace state_history; + std::vector deltas = create_deltas(db, fresh); + state_history_log_header header{.magic = ship_magic(ship_current_version), .block_id = block_state->id}; + + this->write_entry(header, block_state->block->previous, [&deltas](auto& stream) { zlib_pack(stream, deltas); }); +} + +} // namespace eosio diff --git a/libraries/state_history/transaction_trace_cache.cpp b/libraries/state_history/transaction_trace_cache.cpp new file mode 100644 index 00000000000..1fef5486eda --- /dev/null +++ b/libraries/state_history/transaction_trace_cache.cpp @@ -0,0 +1,59 @@ +#include "eosio/state_history/transaction_trace_cache.hpp" + +namespace eosio { +namespace state_history { + +using eosio::chain::packed_transaction; +using eosio::chain::state_history_exception; + + +bool is_onblock(const transaction_trace_ptr& p) { + if (p->action_traces.size() != 1) + return false; + auto& act = p->action_traces[0].act; + using namespace eosio::chain::literals; + if (act.account != eosio::chain::config::system_account_name || act.name != "onblock"_n || + act.authorization.size() != 1) + return false; + auto& auth = act.authorization[0]; + return auth.actor == eosio::chain::config::system_account_name && + auth.permission == eosio::chain::config::active_name; +} + +void transaction_trace_cache::add_transaction(const transaction_trace_ptr& trace, const packed_transaction_ptr& transaction) { + if (trace->receipt) { + if (is_onblock(trace)) + onblock_trace.emplace(trace, transaction); + else if (trace->failed_dtrx_trace) + cached_traces[trace->failed_dtrx_trace->id] = augmented_transaction_trace{trace, transaction}; + else + cached_traces[trace->id] = augmented_transaction_trace{trace, transaction}; + } +} + +std::vector transaction_trace_cache::prepare_traces(const block_state_ptr& block_state) { + + std::vector traces; + if (this->onblock_trace) + traces.push_back(*this->onblock_trace); + for (auto& r : block_state->block->transactions) { + transaction_id_type id; + if (std::holds_alternative(r.trx)) + id = std::get(r.trx); + else + id = std::get(r.trx).id(); + auto it = this->cached_traces.find(id); + EOS_ASSERT(it != this->cached_traces.end() && it->second.trace->receipt, state_history_exception, + "missing trace for transaction ${id}", ("id", id)); + traces.push_back(it->second); + } + clear(); + return traces; +} + +void transaction_trace_cache::clear() { + this->cached_traces.clear(); + this->onblock_trace.reset(); +} + +}} // namespace eosio::state_history diff --git a/libraries/testing/CMakeLists.txt b/libraries/testing/CMakeLists.txt index 0c71359ec44..b2bb8bb9158 100644 --- a/libraries/testing/CMakeLists.txt +++ b/libraries/testing/CMakeLists.txt @@ -1,5 +1,9 @@ file(GLOB HEADERS "include/eosio/testing/*.hpp") +configure_file(contracts.hpp.in include/testing_contracts/contracts.hpp ESCAPE_QUOTES) +add_library(eosio_testing_contracts INTERFACE) +target_include_directories(eosio_testing_contracts INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/include/testing_contracts) + ## SORT .cpp by most likely to change / break compile add_library( eosio_testing tester.cpp @@ -7,7 +11,7 @@ add_library( eosio_testing ${HEADERS} ) -target_link_libraries( eosio_testing eosio_chain fc chainbase Logging IR WAST WASM Runtime Boost::unit_test_framework ) +target_link_libraries( eosio_testing eosio_testing_contracts eosio_chain fc chainbase Logging IR WAST WASM Runtime ) target_include_directories( eosio_testing PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/../wasm-jit/Include" diff --git a/unittests/contracts.hpp.in b/libraries/testing/contracts.hpp.in similarity index 61% rename from unittests/contracts.hpp.in rename to libraries/testing/contracts.hpp.in index 9b7d50114dc..4ea56ec5e1a 100644 --- a/unittests/contracts.hpp.in +++ b/libraries/testing/contracts.hpp.in @@ -1,6 +1,9 @@ #pragma once -#include +#include + +#include +#include #define CORE_SYM_NAME "${CORE_SYMBOL_NAME}" #define CORE_SYM_PRECISION 4 @@ -18,9 +21,17 @@ struct core_sym { }; // CN -> contract C++ name, C -> contract name, D -> top level directory -#define MAKE_READ_WASM_ABI(CN,C, D) \ - static std::vector CN ## _wasm() { return read_wasm("${CMAKE_BINARY_DIR}/unittests/" #D "/" #C "/" #C ".wasm"); } \ - static std::vector CN ## _abi() { return read_abi("${CMAKE_BINARY_DIR}/unittests/" #D "/" #C "/" #C ".abi"); } +#define MAKE_READ_WASM_ABI(CN,C, D) \ + static std::vector CN ## _wasm() { \ + std::string s; \ + fc::read_file_contents("${CMAKE_BINARY_DIR}/unittests/" #D "/" #C "/" #C ".wasm", s); \ + return std::vector(s.begin(), s.end()); \ + } \ + static std::vector CN ## _abi() { \ + std::string s; \ + fc::read_file_contents("${CMAKE_BINARY_DIR}/unittests/" #D "/" #C "/" #C ".abi", s); \ + return std::vector(s.begin(), s.end()); \ + } namespace eosio { namespace testing { @@ -40,6 +51,9 @@ namespace eosio { MAKE_READ_WASM_ABI(deferred_test, deferred_test, test-contracts) MAKE_READ_WASM_ABI(get_sender_test, get_sender_test, test-contracts) MAKE_READ_WASM_ABI(get_table_test, get_table_test, test-contracts) + MAKE_READ_WASM_ABI(get_table_seckey_test, get_table_seckey_test, test-contracts) + MAKE_READ_WASM_ABI(kv_bios, kv_bios, test-contracts) + MAKE_READ_WASM_ABI(kv_test, kv_test, test-contracts) MAKE_READ_WASM_ABI(noop, noop, test-contracts) MAKE_READ_WASM_ABI(payloadless, payloadless, test-contracts) MAKE_READ_WASM_ABI(proxy, proxy, test-contracts) @@ -51,6 +65,11 @@ namespace eosio { MAKE_READ_WASM_ABI(test_api_db, test_api_db, test-contracts) MAKE_READ_WASM_ABI(test_api_multi_index, test_api_multi_index, test-contracts) MAKE_READ_WASM_ABI(test_ram_limit, test_ram_limit, test-contracts) + MAKE_READ_WASM_ABI(action_results, action_results, test-contracts) + MAKE_READ_WASM_ABI(wasm_config_bios, wasm_config_bios, test-contracts) + MAKE_READ_WASM_ABI(params_test, params_test, test-contracts) + MAKE_READ_WASM_ABI(kv_table_test, kv_table_test, test-contracts) + MAKE_READ_WASM_ABI(kv_addr_book, kv_addr_book, test-contracts) }; } /// eosio::testing } /// eosio diff --git a/libraries/testing/include/eosio/testing/backing_store_tester_macros.hpp b/libraries/testing/include/eosio/testing/backing_store_tester_macros.hpp new file mode 100644 index 00000000000..1abae1e8076 --- /dev/null +++ b/libraries/testing/include/eosio/testing/backing_store_tester_macros.hpp @@ -0,0 +1,7 @@ +#ifdef NON_VALIDATING_TEST +#define TESTER tester +#define ROCKSDB_TESTER rocksdb_tester +#else +#define TESTER validating_tester +#define ROCKSDB_TESTER rocksdb_validating_tester +#endif diff --git a/libraries/testing/include/eosio/testing/snapshot_suites.hpp b/libraries/testing/include/eosio/testing/snapshot_suites.hpp new file mode 100644 index 00000000000..adec77a22b5 --- /dev/null +++ b/libraries/testing/include/eosio/testing/snapshot_suites.hpp @@ -0,0 +1,110 @@ +#pragma once + +#include +#include + +using namespace eosio::chain; +using namespace eosio::testing; + +struct variant_snapshot_suite { + using writer_t = variant_snapshot_writer; + using reader_t = variant_snapshot_reader; + using write_storage_t = fc::mutable_variant_object; + using snapshot_t = fc::variant; + + struct writer : public writer_t { + writer( const std::shared_ptr& storage ) + :writer_t(*storage) + ,storage(storage) + { + + } + + std::shared_ptr storage; + }; + + struct reader : public reader_t { + explicit reader(const snapshot_t& storage) + :reader_t(storage) + {} + }; + + + static auto get_writer() { + return std::make_shared(std::make_shared()); + } + + static auto finalize(const std::shared_ptr& w) { + w->finalize(); + return snapshot_t(*w->storage); + } + + static auto get_reader( const snapshot_t& buffer) { + return std::make_shared(buffer); + } + + static snapshot_t load_from_file(const std::string& filename) { + snapshot_input_file file(filename); + return file.read(); + } + + static void write_to_file( const std::string& basename, const snapshot_t& snapshot ) { + snapshot_output_file file(basename); + file.write(snapshot); + } +}; + +struct buffered_snapshot_suite { + using writer_t = ostream_snapshot_writer; + using reader_t = istream_snapshot_reader; + using write_storage_t = std::ostringstream; + using snapshot_t = std::string; + using read_storage_t = std::istringstream; + + struct writer : public writer_t { + writer( const std::shared_ptr& storage ) + :writer_t(*storage) + ,storage(storage) + { + + } + + std::shared_ptr storage; + }; + + struct reader : public reader_t { + explicit reader(const std::shared_ptr& storage) + :reader_t(*storage) + ,storage(storage) + {} + + std::shared_ptr storage; + }; + + + static auto get_writer() { + return std::make_shared(std::make_shared()); + } + + static auto finalize(const std::shared_ptr& w) { + w->finalize(); + return w->storage->str(); + } + + static auto get_reader( const snapshot_t& buffer) { + return std::make_shared(std::make_shared(buffer)); + } + + static snapshot_t load_from_file(const std::string& filename) { + snapshot_input_file file(filename); + return file.read_as_string(); + } + + static void write_to_file( const std::string& basename, const snapshot_t& snapshot ) { + snapshot_output_file file(basename); + file.write(snapshot); + } +}; + +using snapshot_suites = boost::mpl::list; + diff --git a/libraries/testing/include/eosio/testing/tester.hpp b/libraries/testing/include/eosio/testing/tester.hpp index e8e866d361c..7056ce976fe 100644 --- a/libraries/testing/include/eosio/testing/tester.hpp +++ b/libraries/testing/include/eosio/testing/tester.hpp @@ -10,6 +10,7 @@ #include #include +#include #define REQUIRE_EQUAL_OBJECTS(left, right) { auto a = fc::variant( left ); auto b = fc::variant( right ); BOOST_REQUIRE_EQUAL( true, a.is_object() ); \ BOOST_REQUIRE_EQUAL( true, b.is_object() ); \ @@ -62,6 +63,7 @@ namespace eosio { namespace testing { old_bios_only, preactivate_feature_only, preactivate_feature_and_new_bios, + old_wasm_parser, full }; @@ -120,7 +122,7 @@ namespace eosio { namespace testing { char serialized_sig[4096]; datastream sig_ds(serialized_sig, sizeof(serialized_sig)); - fc::raw::pack(sig_ds, (uint8_t)signature::storage_type::position()); + fc::raw::pack(sig_ds, (uint8_t)get_index()); fc::raw::pack(sig_ds, sig); fc::raw::pack(sig_ds, auth_data); fc::raw::pack(sig_ds, json); @@ -152,7 +154,11 @@ namespace eosio { namespace testing { virtual ~base_tester() {}; - void init(const setup_policy policy = setup_policy::full, db_read_mode read_mode = db_read_mode::SPECULATIVE, optional genesis_max_inline_action_size = optional{}, optional config_max_nonprivileged_inline_action_size = optional{}); + void init(const setup_policy policy = setup_policy::full, + db_read_mode read_mode = db_read_mode::SPECULATIVE, + std::optional genesis_max_inline_action_size = std::optional{}, + std::optional config_max_nonprivileged_inline_action_size = std::optional{}, + std::optional config_backing_store = std::optional{}); void init(controller::config config, const snapshot_reader_ptr& snapshot); void init(controller::config config, const genesis_state& genesis); void init(controller::config config); @@ -162,14 +168,13 @@ namespace eosio { namespace testing { void execute_setup_policy(const setup_policy policy); void close(); - template - void open( protocol_feature_set&& pfs, fc::optional expected_chain_id, Lambda lambda ); + void open( protocol_feature_set&& pfs, std::optional expected_chain_id, const std::function& lambda ); void open( protocol_feature_set&& pfs, const snapshot_reader_ptr& snapshot ); void open( protocol_feature_set&& pfs, const genesis_state& genesis ); - void open( protocol_feature_set&& pfs, fc::optional expected_chain_id = {} ); + void open( protocol_feature_set&& pfs, std::optional expected_chain_id = {} ); void open( const snapshot_reader_ptr& snapshot ); void open( const genesis_state& genesis ); - void open( fc::optional expected_chain_id = {} ); + void open( std::optional expected_chain_id = {} ); bool is_same_chain( base_tester& other ); virtual signed_block_ptr produce_block( fc::microseconds skip_time = fc::milliseconds(config::block_interval_ms) ) = 0; @@ -196,8 +201,8 @@ namespace eosio { namespace testing { vector get_scheduled_transactions() const; unapplied_transaction_queue& get_unapplied_transaction_queue() { return unapplied_transactions; } - transaction_trace_ptr push_transaction( packed_transaction& trx, fc::time_point deadline = fc::time_point::maximum(), uint32_t billed_cpu_time_us = DEFAULT_BILLED_CPU_TIME_US ); - transaction_trace_ptr push_transaction( signed_transaction& trx, fc::time_point deadline = fc::time_point::maximum(), uint32_t billed_cpu_time_us = DEFAULT_BILLED_CPU_TIME_US, bool no_throw = false ); + transaction_trace_ptr push_transaction( const packed_transaction& trx, fc::time_point deadline = fc::time_point::maximum(), uint32_t billed_cpu_time_us = DEFAULT_BILLED_CPU_TIME_US ); + transaction_trace_ptr push_transaction( const signed_transaction& trx, fc::time_point deadline = fc::time_point::maximum(), uint32_t billed_cpu_time_us = DEFAULT_BILLED_CPU_TIME_US, bool no_throw = false ); [[nodiscard]] action_result push_action(action&& cert_act, uint64_t authorizer); // TODO/QUESTION: Is this needed? @@ -335,14 +340,14 @@ namespace eosio { namespace testing { static action_result wasm_assert_code( uint64_t error_code ) { return "assertion failure with error code: " + std::to_string(error_code); } auto get_resolver() { - return [this]( const account_name& name ) -> optional { + return [this]( const account_name& name ) -> std::optional { try { const auto& accnt = control->db().get( name ); abi_def abi; if( abi_serializer::to_abi( accnt.abi, abi )) { return abi_serializer( abi, abi_serializer::create_yield_function( abi_serializer_max_time ) ); } - return optional(); + return std::optional(); } FC_RETHROW_EXCEPTIONS( error, "Failed to find or parse ABI for ${name}", ("name", name)) }; } @@ -380,6 +385,7 @@ namespace eosio { namespace testing { void schedule_protocol_features_wo_preactivation(const vector feature_digests); void preactivate_protocol_features(const vector feature_digests); + void preactivate_builtin_protocol_features(const std::vector& features); void preactivate_all_builtin_protocol_features(); static genesis_state default_genesis() { @@ -390,9 +396,13 @@ namespace eosio { namespace testing { return genesis; } - static std::pair default_config(const fc::temp_directory& tempdir, optional genesis_max_inline_action_size = optional{}, optional config_max_nonprivileged_inline_action_size = optional{}) { + static std::pair default_config( + const fc::temp_directory& tempdir, + std::optional genesis_max_inline_action_size = std::optional{}, + std::optional config_max_nonprivileged_inline_action_size = std::optional{}, + std::optional config_backing_store = std::optional{}) { controller::config cfg; - cfg.blocks_dir = tempdir.path() / config::default_blocks_dir_name; + cfg.blog.log_dir = tempdir.path() / config::default_blocks_dir_name; cfg.state_dir = tempdir.path() / config::default_state_dir_name; cfg.state_size = 1024*1024*16; cfg.state_guard_size = 0; @@ -402,9 +412,7 @@ namespace eosio { namespace testing { cfg.eosvmoc_config.cache_size = 1024*1024*8; for(int i = 0; i < boost::unit_test::framework::master_test_suite().argc; ++i) { - if(boost::unit_test::framework::master_test_suite().argv[i] == std::string("--wabt")) - cfg.wasm_runtime = chain::wasm_interface::vm_type::wabt; - else if(boost::unit_test::framework::master_test_suite().argv[i] == std::string("--eos-vm")) + if(boost::unit_test::framework::master_test_suite().argv[i] == std::string("--eos-vm")) cfg.wasm_runtime = chain::wasm_interface::vm_type::eos_vm; else if(boost::unit_test::framework::master_test_suite().argv[i] == std::string("--eos-vm-jit")) cfg.wasm_runtime = chain::wasm_interface::vm_type::eos_vm_jit; @@ -418,6 +426,9 @@ namespace eosio { namespace testing { if (config_max_nonprivileged_inline_action_size) { cfg.max_nonprivileged_inline_action_size = *config_max_nonprivileged_inline_action_size; } + if (config_backing_store) { + cfg.backing_store = *config_backing_store; + } return {cfg, gen}; } @@ -448,8 +459,11 @@ namespace eosio { namespace testing { class tester : public base_tester { public: - tester(setup_policy policy = setup_policy::full, db_read_mode read_mode = db_read_mode::SPECULATIVE, optional genesis_max_inline_action_size = optional{}, optional config_max_nonprivileged_inline_action_size = optional{}) { - init(policy, read_mode, genesis_max_inline_action_size, config_max_nonprivileged_inline_action_size); + tester(setup_policy policy = setup_policy::full, db_read_mode read_mode = db_read_mode::SPECULATIVE, + std::optional genesis_max_inline_action_size = std::optional{}, + std::optional config_max_nonprivileged_inline_action_size = std::optional{}, + std::optional config_backing_store = std::optional{}) { + init(policy, read_mode, genesis_max_inline_action_size, config_max_nonprivileged_inline_action_size, config_backing_store); } tester(controller::config config, const genesis_state& genesis) { @@ -490,6 +504,9 @@ namespace eosio { namespace testing { } } + tester(const std::function& control_setup, setup_policy policy = setup_policy::full, + db_read_mode read_mode = db_read_mode::SPECULATIVE); + using base_tester::produce_block; signed_block_ptr produce_block( fc::microseconds skip_time = fc::milliseconds(config::block_interval_ms) )override { @@ -518,33 +535,27 @@ namespace eosio { namespace testing { try { if( num_blocks_to_producer_before_shutdown > 0 ) produce_blocks( num_blocks_to_producer_before_shutdown ); - if (!skip_validate) - BOOST_REQUIRE_EQUAL( validate(), true ); + if (!skip_validate && std::uncaught_exceptions() == 0) + BOOST_CHECK_EQUAL( validate(), true ); } catch( const fc::exception& e ) { wdump((e.to_detail_string())); } } controller::config vcfg; - validating_tester(const flat_set& trusted_producers = flat_set()) { - auto def_conf = default_config(tempdir); + validating_tester(const flat_set& trusted_producers = flat_set(), + std::optional config_backing_store = std::optional{}); - vcfg = def_conf.first; - config_validator(vcfg); - vcfg.trusted_producers = trusted_producers; - - validating_node = create_validating_node(vcfg, def_conf.second, true); - - init(def_conf.first, def_conf.second); - execute_setup_policy(setup_policy::full); - } + void init_with_trusted_producers(const flat_set& trusted_producers, + std::pair config_state, + std::optional config_backing_store); static void config_validator(controller::config& vcfg) { - FC_ASSERT( vcfg.blocks_dir.filename().generic_string() != "." + FC_ASSERT( vcfg.blog.log_dir.filename().generic_string() != "." && vcfg.state_dir.filename().generic_string() != ".", "invalid path names in controller::config" ); - vcfg.blocks_dir = vcfg.blocks_dir.parent_path() / std::string("v_").append( vcfg.blocks_dir.filename().generic_string() ); - vcfg.state_dir = vcfg.state_dir.parent_path() / std::string("v_").append( vcfg.state_dir.filename().generic_string() ); + vcfg.blog.log_dir = vcfg.blog.log_dir.parent_path() / std::string("v_").append( vcfg.blog.log_dir.filename().generic_string() ); + vcfg.state_dir = vcfg.state_dir.parent_path() / std::string("v_").append( vcfg.state_dir.filename().generic_string() ); vcfg.contracts_console = false; } @@ -553,27 +564,16 @@ namespace eosio { namespace testing { unique_ptr validating_node = std::make_unique(vcfg, make_protocol_feature_set(), genesis.compute_chain_id()); validating_node->add_indices(); if (use_genesis) { - validating_node->startup( []() { return false; }, genesis ); + validating_node->startup( [](){}, []() { return false; }, genesis ); } else { - validating_node->startup( []() { return false; } ); + validating_node->startup( [](){}, []() { return false; } ); } return validating_node; } - validating_tester(const fc::temp_directory& tempdir, bool use_genesis) { - auto def_conf = default_config(tempdir); - vcfg = def_conf.first; - config_validator(vcfg); - validating_node = create_validating_node(vcfg, def_conf.second, use_genesis); - - if (use_genesis) { - init(def_conf.first, def_conf.second); - } - else { - init(def_conf.first); - } - } + validating_tester(const fc::temp_directory& tempdir, bool use_genesis, + std::optional config_backing_store = std::optional{}); template validating_tester(const fc::temp_directory& tempdir, Lambda conf_edit, bool use_genesis) { @@ -591,9 +591,11 @@ namespace eosio { namespace testing { } } + static backing_store_type alternate_type(backing_store_type type); + signed_block_ptr produce_block( fc::microseconds skip_time = fc::milliseconds(config::block_interval_ms) )override { auto sb = _produce_block(skip_time, false); - auto bsf = validating_node->create_block_state_future( sb ); + auto bsf = validating_node->create_block_state_future( sb->calculate_id(), sb ); validating_node->push_block( bsf, forked_branch_callback{}, trx_meta_cache_lookup{} ); return sb; @@ -604,14 +606,14 @@ namespace eosio { namespace testing { } void validate_push_block(const signed_block_ptr& sb) { - auto bs = validating_node->create_block_state_future( sb ); + auto bs = validating_node->create_block_state_future( sb->calculate_id(), sb ); validating_node->push_block( bs, forked_branch_callback{}, trx_meta_cache_lookup{} ); } signed_block_ptr produce_empty_block( fc::microseconds skip_time = fc::milliseconds(config::block_interval_ms) )override { unapplied_transactions.add_aborted( control->abort_block() ); auto sb = _produce_block(skip_time, true); - auto bsf = validating_node->create_block_state_future( sb ); + auto bsf = validating_node->create_block_state_future( sb->calculate_id(), sb ); validating_node->push_block( bsf, forked_branch_callback{}, trx_meta_cache_lookup{} ); return sb; @@ -636,7 +638,7 @@ namespace eosio { namespace testing { validating_node.reset(); validating_node = std::make_unique(vcfg, make_protocol_feature_set(), control->get_chain_id()); validating_node->add_indices(); - validating_node->startup( []() { return false; } ); + validating_node->startup( [](){}, []() { return false; } ); return ok; } @@ -646,6 +648,36 @@ namespace eosio { namespace testing { bool skip_validate = false; }; + class rocksdb_tester : public tester { + public: + rocksdb_tester(setup_policy policy = setup_policy::full, db_read_mode read_mode = db_read_mode::SPECULATIVE, + std::optional genesis_max_inline_action_size = std::optional{}, + std::optional config_max_nonprivileged_inline_action_size = std::optional{}) { + init(policy, read_mode, genesis_max_inline_action_size, config_max_nonprivileged_inline_action_size, + backing_store_type::ROCKSDB); + } + }; + + class rocksdb_validating_tester : public validating_tester { + public: + virtual ~rocksdb_validating_tester() {} + + rocksdb_validating_tester(const flat_set& trusted_producers = flat_set()) + : validating_tester(trusted_producers, {backing_store_type::ROCKSDB}){} + }; + + /** + * Utility predicate to check whether an fc::exception code is equivalent to a given value + */ + struct fc_exception_code_is { + fc_exception_code_is( int64_t code ) + : expected( code ) {} + + bool operator()( const fc::exception& ex ); + + int64_t expected; + }; + /** * Utility predicate to check whether an fc::exception message is equivalent to a given string */ diff --git a/libraries/testing/tester.cpp b/libraries/testing/tester.cpp index ae88d56b737..3388f201170 100644 --- a/libraries/testing/tester.cpp +++ b/libraries/testing/tester.cpp @@ -110,12 +110,12 @@ namespace eosio { namespace testing { protocol_feature_set make_protocol_feature_set(const subjective_restriction_map& custom_subjective_restrictions) { protocol_feature_set pfs; - map< builtin_protocol_feature_t, optional > visited_builtins; + map< builtin_protocol_feature_t, std::optional > visited_builtins; std::function add_builtins = [&pfs, &visited_builtins, &add_builtins, &custom_subjective_restrictions] ( builtin_protocol_feature_t codename ) -> digest_type { - auto res = visited_builtins.emplace( codename, optional() ); + auto res = visited_builtins.emplace( codename, std::optional() ); if( !res.second ) { EOS_ASSERT( res.first->second, protocol_feature_exception, "invariant failure: cycle found in builtin protocol feature dependencies" @@ -150,8 +150,12 @@ namespace eosio { namespace testing { return control->head_block_id() == other.control->head_block_id(); } - void base_tester::init(const setup_policy policy, db_read_mode read_mode, optional genesis_max_inline_action_size, optional config_max_nonprivileged_inline_action_size) { - auto def_conf = default_config(tempdir, genesis_max_inline_action_size, config_max_nonprivileged_inline_action_size); + void base_tester::init(const setup_policy policy, db_read_mode read_mode, + std::optional genesis_max_inline_action_size, + std::optional config_max_nonprivileged_inline_action_size, + std::optional config_backing_store) { + auto def_conf = default_config(tempdir, genesis_max_inline_action_size, + config_max_nonprivileged_inline_action_size, config_backing_store); def_conf.first.read_mode = read_mode; cfg = def_conf.first; @@ -214,6 +218,29 @@ namespace eosio { namespace testing { set_before_producer_authority_bios_contract(); break; } + case setup_policy::old_wasm_parser: { + schedule_preactivate_protocol_feature(); + produce_block(); + set_before_producer_authority_bios_contract(); + preactivate_builtin_protocol_features({ + builtin_protocol_feature_t::only_link_to_existing_permission, + builtin_protocol_feature_t::replace_deferred, + builtin_protocol_feature_t::no_duplicate_deferred_id, + builtin_protocol_feature_t::fix_linkauth_restriction, + builtin_protocol_feature_t::disallow_empty_producer_schedule, + builtin_protocol_feature_t::restrict_action_to_self, + builtin_protocol_feature_t::only_bill_first_authorizer, + builtin_protocol_feature_t::forward_setcode, + builtin_protocol_feature_t::get_sender, + builtin_protocol_feature_t::ram_restrictions, + builtin_protocol_feature_t::webauthn_key, + builtin_protocol_feature_t::wtmsig_block_signatures, + builtin_protocol_feature_t::kv_database + }); + produce_block(); + set_bios_contract(); + break; + } case setup_policy::full: { schedule_preactivate_protocol_feature(); produce_block(); @@ -242,17 +269,16 @@ namespace eosio { namespace testing { open( make_protocol_feature_set(), genesis ); } - void base_tester::open( fc::optional expected_chain_id ) { + void base_tester::open( std::optional expected_chain_id ) { open( make_protocol_feature_set(), expected_chain_id ); } - template - void base_tester::open( protocol_feature_set&& pfs, fc::optional expected_chain_id, Lambda lambda ) { + void base_tester::open( protocol_feature_set&& pfs, std::optional expected_chain_id, const std::function& lambda ) { if( !expected_chain_id ) { expected_chain_id = controller::extract_chain_id_from_db( cfg.state_dir ); if( !expected_chain_id ) { - if( fc::is_regular_file( cfg.blocks_dir / "blocks.log" ) ) { - expected_chain_id = block_log::extract_chain_id( cfg.blocks_dir ); + if( fc::is_regular_file( cfg.blog.log_dir / "blocks.log" ) ) { + expected_chain_id = block_log::extract_chain_id( cfg.blog.log_dir ); } else { expected_chain_id = genesis_state().compute_chain_id(); } @@ -261,17 +287,17 @@ namespace eosio { namespace testing { control.reset( new controller(cfg, std::move(pfs), *expected_chain_id) ); control->add_indices(); - lambda(); + if (lambda) lambda(); chain_transactions.clear(); control->accepted_block.connect([this]( const block_state_ptr& block_state ){ FC_ASSERT( block_state->block ); - for( const auto& receipt : block_state->block->transactions ) { - if( receipt.trx.contains() ) { - auto &pt = receipt.trx.get(); - chain_transactions[pt.get_transaction().id()] = receipt; + for( auto receipt : block_state->block->transactions ) { + if( std::holds_alternative(receipt.trx) ) { + auto &pt = std::get(receipt.trx); + chain_transactions[pt.get_transaction().id()] = std::move(receipt); } else { - auto& id = receipt.trx.get(); - chain_transactions[id] = receipt; + auto& id = std::get(receipt.trx); + chain_transactions[id] = std::move(receipt); } } }); @@ -281,24 +307,24 @@ namespace eosio { namespace testing { const auto& snapshot_chain_id = controller::extract_chain_id( *snapshot ); snapshot->return_to_header(); open(std::move(pfs), snapshot_chain_id, [&snapshot,&control=this->control]() { - control->startup([]() { return false; }, snapshot ); + control->startup( [](){}, []() { return false; }, snapshot ); }); } void base_tester::open( protocol_feature_set&& pfs, const genesis_state& genesis ) { open(std::move(pfs), genesis.compute_chain_id(), [&genesis,&control=this->control]() { - control->startup( []() { return false; }, genesis ); + control->startup( [](){}, []() { return false; }, genesis ); }); } - void base_tester::open( protocol_feature_set&& pfs, fc::optional expected_chain_id ) { + void base_tester::open( protocol_feature_set&& pfs, std::optional expected_chain_id ) { open(std::move(pfs), expected_chain_id, [&control=this->control]() { - control->startup( []() { return false; } ); + control->startup( [](){}, []() { return false; } ); }); } void base_tester::push_block(signed_block_ptr b) { - auto bsf = control->create_block_state_future(b); + auto bsf = control->create_block_state_future(b->calculate_id(), b); unapplied_transactions.add_aborted( control->abort_block() ); control->push_block( bsf, [this]( const branch_type& forked_branch ) { unapplied_transactions.add_forked( forked_branch ); @@ -307,8 +333,8 @@ namespace eosio { namespace testing { } ); auto itr = last_produced_block.find(b->producer); - if (itr == last_produced_block.end() || block_header::num_from_id(b->id()) > block_header::num_from_id(itr->second)) { - last_produced_block[b->producer] = b->id(); + if (itr == last_produced_block.end() || b->block_num() > block_header::num_from_id(itr->second)) { + last_produced_block[b->producer] = b->calculate_id(); } } @@ -536,7 +562,7 @@ namespace eosio { namespace testing { return push_transaction( trx ); } - transaction_trace_ptr base_tester::push_transaction( packed_transaction& trx, + transaction_trace_ptr base_tester::push_transaction( const packed_transaction& trx, fc::time_point deadline, uint32_t billed_cpu_time_us ) @@ -555,7 +581,7 @@ namespace eosio { namespace testing { return r; } FC_RETHROW_EXCEPTIONS( warn, "transaction_header: ${header}", ("header", transaction_header(trx.get_transaction()) )) } - transaction_trace_ptr base_tester::push_transaction( signed_transaction& trx, + transaction_trace_ptr base_tester::push_transaction( const signed_transaction& trx, fc::time_point deadline, uint32_t billed_cpu_time_us, bool no_throw @@ -572,8 +598,8 @@ namespace eosio { namespace testing { auto time_limit = deadline == fc::time_point::maximum() ? fc::microseconds::maximum() : fc::microseconds( deadline - fc::time_point::now() ); - auto ptrx = std::make_shared( trx, c ); - auto fut = transaction_metadata::start_recover_keys( ptrx, control->get_thread_pool(), control->get_chain_id(), time_limit ); + auto ptrx = std::make_shared( signed_transaction(trx), true, c ); + auto fut = transaction_metadata::start_recover_keys( std::move( ptrx ), control->get_thread_pool(), control->get_chain_id(), time_limit ); auto r = control->push_transaction( fut.get(), deadline, billed_cpu_time_us, billed_cpu_time_us > 0, 0 ); if (no_throw) return r; if( r->except_ptr ) std::rethrow_exception( r->except_ptr ); @@ -673,7 +699,7 @@ namespace eosio { namespace testing { } FC_CAPTURE_AND_RETHROW() } transaction_trace_ptr base_tester::push_reqauth( account_name from, const vector& auths, const vector& keys ) { - variant pretty_trx = fc::mutable_variant_object() + fc::variant pretty_trx = fc::mutable_variant_object() ("actions", fc::variants({ fc::mutable_variant_object() ("account", name(config::system_account_name)) @@ -707,7 +733,7 @@ namespace eosio { namespace testing { transaction_trace_ptr base_tester::push_dummy(account_name from, const string& v, uint32_t billed_cpu_time_us) { // use reqauth for a normal action, this could be anything - variant pretty_trx = fc::mutable_variant_object() + fc::variant pretty_trx = fc::mutable_variant_object() ("actions", fc::variants({ fc::mutable_variant_object() ("account", name(config::system_account_name)) @@ -746,7 +772,7 @@ namespace eosio { namespace testing { transaction_trace_ptr base_tester::transfer( account_name from, account_name to, asset amount, string memo, account_name currency ) { - variant pretty_trx = fc::mutable_variant_object() + fc::variant pretty_trx = fc::mutable_variant_object() ("actions", fc::variants({ fc::mutable_variant_object() ("account", currency) @@ -775,7 +801,7 @@ namespace eosio { namespace testing { transaction_trace_ptr base_tester::issue( account_name to, string amount, account_name currency, string memo ) { - variant pretty_trx = fc::mutable_variant_object() + fc::variant pretty_trx = fc::mutable_variant_object() ("actions", fc::variants({ fc::mutable_variant_object() ("account", currency) @@ -944,7 +970,7 @@ namespace eosio { namespace testing { const symbol& asset_symbol, const account_name& account ) const { const auto& db = control->db(); - const auto* tbl = db.template find(boost::make_tuple(code, account, N(accounts))); + const auto* tbl = db.template find(boost::make_tuple(code, account, "accounts"_n)); share_type result = 0; // the balance is implied to be 0 if either the table or row does not exist @@ -1026,7 +1052,7 @@ namespace eosio { namespace testing { auto block = a.control->fetch_block_by_number(i); if( block ) { //&& !b.control->is_known_block(block->id()) ) { - auto bsf = b.control->create_block_state_future( block ); + auto bsf = b.control->create_block_state_future( block->calculate_id(), block ); b.control->abort_block(); b.control->push_block(bsf, forked_branch_callback{}, trx_meta_cache_lookup{}); //, eosio::chain::validation_steps::created_block); } @@ -1076,7 +1102,7 @@ namespace eosio { namespace testing { schedule_variant.emplace_back(e.get_abi_variant()); } - return push_action( config::system_account_name, N(setprods), config::system_account_name, + return push_action( config::system_account_name, "setprods"_n, config::system_account_name, fc::mutable_variant_object()("schedule", schedule_variant)); } @@ -1088,12 +1114,12 @@ namespace eosio { namespace testing { vector legacy_keys; legacy_keys.reserve(schedule.size()); for (const auto &p : schedule) { - p.authority.visit([&legacy_keys, &p](const auto& auth){ + std::visit([&legacy_keys, &p](const auto& auth){ legacy_keys.emplace_back(legacy::producer_key{p.producer_name, auth.keys.front().key}); - }); + }, p.authority); } - return push_action( config::system_account_name, N(setprods), config::system_account_name, + return push_action( config::system_account_name, "setprods"_n, config::system_account_name, fc::mutable_variant_object()("schedule", legacy_keys)); } @@ -1114,11 +1140,24 @@ namespace eosio { namespace testing { void base_tester::preactivate_protocol_features(const vector feature_digests) { for( const auto& feature_digest: feature_digests ) { - push_action( config::system_account_name, N(activate), config::system_account_name, + push_action( config::system_account_name, "activate"_n, config::system_account_name, fc::mutable_variant_object()("feature_digest", feature_digest) ); } } + void base_tester::preactivate_builtin_protocol_features(const std::vector& builtin_features) { + const auto& pfs = control->get_protocol_feature_manager().get_protocol_feature_set(); + + // This behavior is disabled by configurable_wasm_limits + std::vector features; + for(builtin_protocol_feature_t feature : builtin_features ) { + if( auto digest = pfs.get_builtin_digest( feature ) ) { + features.push_back( *digest ); + } + } + preactivate_protocol_features(features); + } + void base_tester::preactivate_all_builtin_protocol_features() { const auto& pfm = control->get_protocol_feature_manager(); const auto& pfs = pfm.get_protocol_feature_set(); @@ -1156,6 +1195,75 @@ namespace eosio { namespace testing { preactivate_protocol_features( preactivations ); } + tester::tester(const std::function& control_setup, setup_policy policy, db_read_mode read_mode) { + auto def_conf = default_config(tempdir); + def_conf.first.read_mode = read_mode; + cfg = def_conf.first; + + base_tester::open(make_protocol_feature_set(), def_conf.second.compute_chain_id(), + [&genesis = def_conf.second, &control = this->control, &control_setup]() { + control_setup(*control); + control->startup([]() {}, []() { return false; }, genesis); + }); + + execute_setup_policy(policy); + } + + validating_tester::validating_tester(const flat_set& trusted_producers, + std::optional config_backing_store) { + auto def_conf = default_config(tempdir, std::optional{}, std::optional{}, config_backing_store); + init_with_trusted_producers(trusted_producers, def_conf, config_backing_store); + } + + void validating_tester::init_with_trusted_producers(const flat_set& trusted_producers, + std::pair config_state, + std::optional config_backing_store) { + vcfg = config_state.first; + if (config_backing_store) { + // can only have one instance of RocksDB running in the test process + vcfg.backing_store = alternate_type(*config_backing_store); + } + config_validator(vcfg); + vcfg.trusted_producers = trusted_producers; + + validating_node = create_validating_node(vcfg, config_state.second, true); + + init(config_state.first, config_state.second); + execute_setup_policy(setup_policy::full); + } + + validating_tester::validating_tester(const fc::temp_directory& tempdir, bool use_genesis, + std::optional config_backing_store) { + auto def_conf = default_config(tempdir, std::optional{}, std::optional{}, config_backing_store); + vcfg = def_conf.first; + if (config_backing_store) { + // can only have one instance of RocksDB running in the test process + vcfg.backing_store = alternate_type(*config_backing_store); + } + config_validator(vcfg); + validating_node = create_validating_node(vcfg, def_conf.second, use_genesis); + + if (use_genesis) { + init(def_conf.first, def_conf.second); + } + else { + init(def_conf.first); + } + } + + backing_store_type validating_tester::alternate_type(backing_store_type type) { + return type == backing_store_type::CHAINBASE ? backing_store_type::ROCKSDB : backing_store_type::CHAINBASE; + } + + bool fc_exception_code_is::operator()( const fc::exception& ex ) { + bool match = (ex.code() == expected); + if( !match ) { + auto message = ex.get_log().at( 0 ).get_message(); + BOOST_TEST_MESSAGE( "LOG: expected code: " << expected << ", actual code: " << ex.code() << ", message: " << message ); + } + return match; + } + bool fc_exception_message_is::operator()( const fc::exception& ex ) { auto message = ex.get_log().at( 0 ).get_message(); bool match = (message == expected); diff --git a/libraries/wabt b/libraries/wabt deleted file mode 160000 index 73432482e43..00000000000 --- a/libraries/wabt +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 73432482e435e2ddf0eaeb0632641592d9efd35a diff --git a/libraries/wasm-jit/CMakeLists.txt b/libraries/wasm-jit/CMakeLists.txt index 7697f6ee59f..a62632b2623 100644 --- a/libraries/wasm-jit/CMakeLists.txt +++ b/libraries/wasm-jit/CMakeLists.txt @@ -69,7 +69,6 @@ add_subdirectory(Include/Inline) add_subdirectory(Source/IR) add_subdirectory(Source/Logging) -add_subdirectory(Source/Platform) add_subdirectory(Source/Runtime) add_subdirectory(Source/WASM) add_subdirectory(Source/WAST) diff --git a/libraries/wasm-jit/Include/Inline/Serialization.h b/libraries/wasm-jit/Include/Inline/Serialization.h index f9a92ce0794..57b78cad421 100644 --- a/libraries/wasm-jit/Include/Inline/Serialization.h +++ b/libraries/wasm-jit/Include/Inline/Serialization.h @@ -8,6 +8,10 @@ #include #include +namespace WASM +{ + extern bool check_limits; +} namespace Serialization { // An exception that is thrown for various errors during serialization. @@ -267,7 +271,7 @@ namespace Serialization // Advance the stream before resizing the string: // try to get a serialization exception before making a huge allocation for malformed input. const U8* inputBytes = stream.advance(size); - if (size >= max_size) + if (size >= max_size && WASM::check_limits) throw FatalSerializationException(std::string("Trying to deserialize string of size : " + std::to_string((uint64_t)size) + ", which is over by "+std::to_string(size - max_size )+" bytes")); string.resize(size); memcpy(const_cast(string.data()),inputBytes,size); @@ -287,7 +291,7 @@ namespace Serialization // Grow the vector one element at a time: // try to get a serialization exception before making a huge allocation for malformed input. vector.clear(); - if (size >= max_size) + if (size >= max_size && WASM::check_limits) throw FatalSerializationException(std::string("Trying to deserialize array of size : " + std::to_string((uint64_t)size) + ", which is over by "+std::to_string(size - max_size )+" bytes")); for(Uptr index = 0;index < size;++index) { diff --git a/libraries/wasm-jit/Include/Platform/Platform.h b/libraries/wasm-jit/Include/Platform/Platform.h index edaeef071d8..8c964d8487b 100644 --- a/libraries/wasm-jit/Include/Platform/Platform.h +++ b/libraries/wasm-jit/Include/Platform/Platform.h @@ -6,23 +6,13 @@ #include "Inline/BasicTypes.h" #include "Inline/Errors.h" -#ifdef _WIN32 - #define THREAD_LOCAL thread_local - #define DLL_EXPORT __declspec(dllexport) - #define DLL_IMPORT __declspec(dllimport) - #define FORCEINLINE __forceinline - #define SUPPRESS_UNUSED(variable) (void)(variable); - #include - #define PACKED_STRUCT(definition) __pragma(pack(push, 1)) definition; __pragma(pack(pop)) -#else - // Use __thread instead of the C++11 thread_local because Apple's clang doesn't support thread_local yet. - #define THREAD_LOCAL __thread - #define DLL_EXPORT - #define DLL_IMPORT - #define FORCEINLINE inline __attribute__((always_inline)) - #define SUPPRESS_UNUSED(variable) (void)(variable); - #define PACKED_STRUCT(definition) definition __attribute__((packed)); -#endif +// Use __thread instead of the C++11 thread_local because Apple's clang doesn't support thread_local yet. +#define THREAD_LOCAL __thread +#define DLL_EXPORT +#define DLL_IMPORT +#define FORCEINLINE inline __attribute__((always_inline)) +#define SUPPRESS_UNUSED(variable) (void)(variable); +#define PACKED_STRUCT(definition) definition __attribute__((packed)); #ifndef PLATFORM_API #define PLATFORM_API DLL_IMPORT @@ -31,155 +21,13 @@ namespace Platform { // countLeadingZeroes/countTrailingZeroes returns the number of leading/trailing zeroes, or the bit width of the input if no bits are set. - #ifdef _WIN32 - // BitScanReverse/BitScanForward return 0 if the input is 0. - inline U32 countLeadingZeroes(U32 value) { unsigned long result; return _BitScanReverse(&result,value) ? (31 - result) : 32; } - inline U32 countTrailingZeroes(U32 value) { unsigned long result; return _BitScanForward(&result,value) ? result : 32; } - - #ifdef _WIN64 - inline U64 countLeadingZeroes(U64 value) { unsigned long result; return _BitScanReverse64(&result,value) ? (63 - result) : 64; } - inline U64 countTrailingZeroes(U64 value) { unsigned long result; return _BitScanForward64(&result,value) ? result : 64; } - #else - inline U64 countLeadingZeroes(U64 value) { throw; } - inline U64 countTrailingZeroes(U64 value) { throw; } - #endif - #else - // __builtin_clz/__builtin_ctz are undefined if the input is 0. - inline U64 countLeadingZeroes(U64 value) { return value == 0 ? 64 : __builtin_clzll(value); } - inline U32 countLeadingZeroes(U32 value) { return value == 0 ? 32 : __builtin_clz(value); } - inline U64 countTrailingZeroes(U64 value) { return value == 0 ? 64 : __builtin_ctzll(value); } - inline U32 countTrailingZeroes(U32 value) { return value == 0 ? 32 : __builtin_ctz(value); } - #endif + // __builtin_clz/__builtin_ctz are undefined if the input is 0. + inline U64 countLeadingZeroes(U64 value) { return value == 0 ? 64 : __builtin_clzll(value); } + inline U32 countLeadingZeroes(U32 value) { return value == 0 ? 32 : __builtin_clz(value); } + inline U64 countTrailingZeroes(U64 value) { return value == 0 ? 64 : __builtin_ctzll(value); } + inline U32 countTrailingZeroes(U32 value) { return value == 0 ? 32 : __builtin_ctz(value); } inline U64 floorLogTwo(U64 value) { return value <= 1 ? 0 : 63 - countLeadingZeroes(value); } inline U32 floorLogTwo(U32 value) { return value <= 1 ? 0 : 31 - countLeadingZeroes(value); } inline U64 ceilLogTwo(U64 value) { return floorLogTwo(value * 2 - 1); } inline U32 ceilLogTwo(U32 value) { return floorLogTwo(value * 2 - 1); } - - // - // Memory - // - - // Describes allowed memory accesses. - enum class MemoryAccess - { - None, - ReadOnly, - ReadWrite, - Execute, - ReadWriteExecute - }; - - // Returns the base 2 logarithm of the smallest virtual page size. - PLATFORM_API Uptr getPageSizeLog2(); - - // Allocates virtual addresses without commiting physical pages to them. - // Returns the base virtual address of the allocated addresses, or nullptr if the virtual address space has been exhausted. - PLATFORM_API U8* allocateVirtualPages(Uptr numPages); - - // Commits physical memory to the specified virtual pages. - // baseVirtualAddress must be a multiple of the preferred page size. - // Return true if successful, or false if physical memory has been exhausted. - PLATFORM_API bool commitVirtualPages(U8* baseVirtualAddress,Uptr numPages,MemoryAccess access = MemoryAccess::ReadWrite); - - // Changes the allowed access to the specified virtual pages. - // baseVirtualAddress must be a multiple of the preferred page size. - // Return true if successful, or false if the access-level could not be set. - PLATFORM_API bool setVirtualPageAccess(U8* baseVirtualAddress,Uptr numPages,MemoryAccess access); - - // Decommits the physical memory that was committed to the specified virtual pages. - // baseVirtualAddress must be a multiple of the preferred page size. - PLATFORM_API void decommitVirtualPages(U8* baseVirtualAddress,Uptr numPages); - - // Frees virtual addresses. Any physical memory committed to the addresses must have already been decommitted. - // baseVirtualAddress must be a multiple of the preferred page size. - PLATFORM_API void freeVirtualPages(U8* baseVirtualAddress,Uptr numPages); - - // - // Call stack and exceptions - // - - // Describes a call stack. - struct CallStack - { - struct Frame - { - Uptr ip; - }; - std::vector stackFrames; - }; - - // Captures the execution context of the caller. - PLATFORM_API CallStack captureCallStack(Uptr numOmittedFramesFromTop = 0); - - // Describes an instruction pointer. - PLATFORM_API bool describeInstructionPointer(Uptr ip,std::string& outDescription); - - #ifdef _WIN64 - // Registers/deregisters the data used by Windows SEH to unwind stack frames. - PLATFORM_API void registerSEHUnwindInfo(Uptr imageLoadAddress,Uptr pdataAddress,Uptr pdataNumBytes); - PLATFORM_API void deregisterSEHUnwindInfo(Uptr pdataAddress); - #endif - - // Calls a thunk, and if it causes any of some specific hardware traps, returns true. - // If a trap was caught, the outCause, outContext, and outOperand parameters are set to describe the trap. - enum HardwareTrapType - { - none, - accessViolation, - stackOverflow, - intDivideByZeroOrOverflow - }; - PLATFORM_API HardwareTrapType catchHardwareTraps( - CallStack& outTrapCallStack, - Uptr& outTrapOperand, - const std::function& thunk - ); - PLATFORM_API [[noreturn]] void immediately_exit(std::exception_ptr except); - - // - // Threading - // - - // Returns the current value of a clock that may be used as an absolute time for wait timeouts. - // The resolution is microseconds, and the origin is arbitrary. - PLATFORM_API U64 getMonotonicClock(); - - // Platform-independent mutexes. - struct Mutex; - PLATFORM_API Mutex* createMutex(); - PLATFORM_API void destroyMutex(Mutex* mutex); - PLATFORM_API void lockMutex(Mutex* mutex); - PLATFORM_API void unlockMutex(Mutex* mutex); - - // RAII-style lock for Mutex. - struct Lock - { - Lock(Mutex* inMutex) : mutex(inMutex) { lockMutex(mutex); } - ~Lock() { unlockMutex(mutex); } - - void release() - { - if(mutex) - { - unlockMutex(mutex); - } - mutex = nullptr; - } - - void detach() - { - WAVM_ASSERT_THROW(mutex); - mutex = nullptr; - } - - private: - Mutex* mutex; - }; - - // Platform-independent events. - struct Event; - PLATFORM_API Event* createEvent(); - PLATFORM_API void destroyEvent(Event* event); - PLATFORM_API bool waitForEvent(Event* event,U64 untilClock); - PLATFORM_API void signalEvent(Event* event); } diff --git a/libraries/wasm-jit/Include/Runtime/Linker.h b/libraries/wasm-jit/Include/Runtime/Linker.h index b9b75b6b935..eb02588c882 100644 --- a/libraries/wasm-jit/Include/Runtime/Linker.h +++ b/libraries/wasm-jit/Include/Runtime/Linker.h @@ -40,7 +40,7 @@ namespace Runtime bool resolve(const std::string& moduleName,const std::string& exportName,IR::ObjectType type,Runtime::ObjectInstance*& outObject) override { if(!innerResolver) { innerResolver = innerResolverThunk(); } - return innerResolver->resolve(moduleName,exportName,type,outObject); + return innerResolver->resolve(moduleName,exportName,type,outObject); } private: @@ -54,7 +54,7 @@ namespace Runtime { bool resolve(const std::string& moduleName,const std::string& exportName,IR::ObjectType type,Runtime::ObjectInstance*& outObject) override { - return false; + return false; } }; @@ -75,4 +75,4 @@ namespace Runtime }; RUNTIME_API LinkResult linkModule(const IR::Module& module,Resolver& resolver); -} \ No newline at end of file +} diff --git a/libraries/wasm-jit/Include/WASM/WASM.h b/libraries/wasm-jit/Include/WASM/WASM.h index 458e0155f85..777f37b8fcb 100644 --- a/libraries/wasm-jit/Include/WASM/WASM.h +++ b/libraries/wasm-jit/Include/WASM/WASM.h @@ -11,6 +11,11 @@ namespace Serialization { struct InputStream; struct OutputStream; } namespace WASM { + extern bool check_limits; + struct scoped_skip_checks { + scoped_skip_checks() { check_limits = false; } + ~scoped_skip_checks() { check_limits = true; } + }; WEBASSEMBLY_API void serialize(Serialization::InputStream& stream,IR::Module& module); WEBASSEMBLY_API void serialize(Serialization::OutputStream& stream,const IR::Module& module); } diff --git a/libraries/wasm-jit/Source/Logging/CMakeLists.txt b/libraries/wasm-jit/Source/Logging/CMakeLists.txt index 7ca6fff5842..659dd7c9ba5 100644 --- a/libraries/wasm-jit/Source/Logging/CMakeLists.txt +++ b/libraries/wasm-jit/Source/Logging/CMakeLists.txt @@ -7,7 +7,6 @@ include_directories(${WAVM_INCLUDE_DIR}/Logging) add_definitions(-DLOGGING_API=DLL_EXPORT) add_library(Logging STATIC ${Sources} ${PublicHeaders}) -target_link_libraries(Logging Platform) # Link with dl on Linux for dladdr. if(CMAKE_SYSTEM_NAME STREQUAL "Linux") diff --git a/libraries/wasm-jit/Source/Logging/Logging.cpp b/libraries/wasm-jit/Source/Logging/Logging.cpp index a18f6225348..7fae383da9a 100644 --- a/libraries/wasm-jit/Source/Logging/Logging.cpp +++ b/libraries/wasm-jit/Source/Logging/Logging.cpp @@ -7,7 +7,6 @@ namespace Log { - static Platform::Mutex* categoryEnabledMutex = Platform::createMutex(); static bool categoryEnabled[(Uptr)Category::num] = { true, // error @@ -20,19 +19,16 @@ namespace Log }; void setCategoryEnabled(Category category,bool enable) { - Platform::Lock lock(categoryEnabledMutex); WAVM_ASSERT_THROW(category < Category::num); categoryEnabled[(Uptr)category] = enable; } bool isCategoryEnabled(Category category) { - Platform::Lock lock(categoryEnabledMutex); WAVM_ASSERT_THROW(category < Category::num); return categoryEnabled[(Uptr)category]; } void printf(Category category,const char* format,...) { - Platform::Lock lock(categoryEnabledMutex); if(categoryEnabled[(Uptr)category]) { va_list varArgs; diff --git a/libraries/wasm-jit/Source/Platform/CMakeLists.txt b/libraries/wasm-jit/Source/Platform/CMakeLists.txt deleted file mode 100644 index d77fe151391..00000000000 --- a/libraries/wasm-jit/Source/Platform/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -set(Sources - POSIX.cpp - Windows.cpp) -set(PublicHeaders - ${WAVM_INCLUDE_DIR}/Platform/Platform.h) -include_directories(${WAVM_INCLUDE_DIR}/Platform) - -add_definitions(-DPLATFORM_API=DLL_EXPORT) - -add_library(Platform STATIC ${Sources} ${PublicHeaders}) - -# Link with dl on Linux for dladdr. -if(CMAKE_SYSTEM_NAME STREQUAL "Linux") - target_link_libraries(Platform dl pthread rt) -endif() - -install(TARGETS Platform - LIBRARY DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}) - diff --git a/libraries/wasm-jit/Source/Platform/POSIX.cpp b/libraries/wasm-jit/Source/Platform/POSIX.cpp deleted file mode 100644 index b6810e97480..00000000000 --- a/libraries/wasm-jit/Source/Platform/POSIX.cpp +++ /dev/null @@ -1,422 +0,0 @@ -#ifndef _WIN32 - -#include "Inline/BasicTypes.h" -#include "Inline/Errors.h" -#include "Platform/Platform.h" - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include - -#define __STDC_FORMAT_MACROS -#include - -#ifdef __APPLE__ - #define MAP_ANONYMOUS MAP_ANON -#endif - -#ifdef __linux__ - #include - #include -#endif -#ifdef __FreeBSD__ - #include - #include - #include - #include -#endif - -namespace Platform -{ - static Uptr internalGetPreferredVirtualPageSizeLog2() - { - U32 preferredVirtualPageSize = sysconf(_SC_PAGESIZE); - // Verify our assumption that the virtual page size is a power of two. - WAVM_ASSERT_THROW(!(preferredVirtualPageSize & (preferredVirtualPageSize - 1))); - return floorLogTwo(preferredVirtualPageSize); - } - Uptr getPageSizeLog2() - { - static Uptr preferredVirtualPageSizeLog2 = internalGetPreferredVirtualPageSizeLog2(); - return preferredVirtualPageSizeLog2; - } - - U32 memoryAccessAsPOSIXFlag(MemoryAccess access) - { - switch(access) - { - default: - case MemoryAccess::None: return PROT_NONE; - case MemoryAccess::ReadOnly: return PROT_READ; - case MemoryAccess::ReadWrite: return PROT_READ | PROT_WRITE; - case MemoryAccess::Execute: return PROT_EXEC; - case MemoryAccess::ReadWriteExecute: return PROT_EXEC | PROT_READ | PROT_WRITE; - } - } - - bool isPageAligned(U8* address) - { - const Uptr addressBits = reinterpret_cast(address); - return (addressBits & ((1ull << getPageSizeLog2()) - 1)) == 0; - } - - U8* allocateVirtualPages(Uptr numPages) - { - Uptr numBytes = numPages << getPageSizeLog2(); - auto result = mmap(nullptr,numBytes,PROT_NONE,MAP_PRIVATE | MAP_ANONYMOUS,-1,0); - if(result == MAP_FAILED) - { - return nullptr; - } - return (U8*)result; - } - - bool commitVirtualPages(U8* baseVirtualAddress,Uptr numPages,MemoryAccess access) - { - errorUnless(isPageAligned(baseVirtualAddress)); - return mprotect(baseVirtualAddress,numPages << getPageSizeLog2(),memoryAccessAsPOSIXFlag(access)) == 0; - } - - bool setVirtualPageAccess(U8* baseVirtualAddress,Uptr numPages,MemoryAccess access) - { - errorUnless(isPageAligned(baseVirtualAddress)); - return mprotect(baseVirtualAddress,numPages << getPageSizeLog2(),memoryAccessAsPOSIXFlag(access)) == 0; - } - - void decommitVirtualPages(U8* baseVirtualAddress,Uptr numPages) - { - errorUnless(isPageAligned(baseVirtualAddress)); - auto numBytes = numPages << getPageSizeLog2(); - if(madvise(baseVirtualAddress,numBytes,MADV_DONTNEED)) { Errors::fatal("madvise failed"); } - if(mprotect(baseVirtualAddress,numBytes,PROT_NONE)) { Errors::fatal("mprotect failed"); } - } - - void freeVirtualPages(U8* baseVirtualAddress,Uptr numPages) - { - errorUnless(isPageAligned(baseVirtualAddress)); - if(munmap(baseVirtualAddress,numPages << getPageSizeLog2())) { Errors::fatal("munmap failed"); } - } - - bool describeInstructionPointer(Uptr ip,std::string& outDescription) - { - #if defined __linux__ || defined __FreeBSD__ - // Look up static symbol information for the address. - Dl_info symbolInfo; - if(dladdr((void*)ip,&symbolInfo) && symbolInfo.dli_sname) - { - outDescription = symbolInfo.dli_sname; - return true; - } - #endif - return false; - } - - enum { signalStackNumBytes = 65536 }; - THREAD_LOCAL U8* signalStack = nullptr; - THREAD_LOCAL U8* stackMinAddr = nullptr; - THREAD_LOCAL U8* stackMaxAddr = nullptr; - - void initThread() - { - if(!signalStack) - { - // Allocate a stack to use when handling signals, so stack overflow can be handled safely. - signalStack = new U8[signalStackNumBytes]; - stack_t signalStackInfo; - signalStackInfo.ss_size = signalStackNumBytes; - signalStackInfo.ss_sp = signalStack; - signalStackInfo.ss_flags = 0; - if(sigaltstack(&signalStackInfo,nullptr) < 0) - { - Errors::fatal("sigaltstack failed"); - } - - // Get the stack address from pthreads, but use getrlimit to find the maximum size of the stack instead of the current. - struct rlimit stackLimit; - getrlimit(RLIMIT_STACK,&stackLimit); - #if defined __linux__ || defined __FreeBSD__ - // Linux uses pthread_getattr_np/pthread_attr_getstack, and returns a pointer to the minimum address of the stack. - pthread_attr_t threadAttributes; - pthread_attr_init(&threadAttributes); - #ifdef __linux__ - pthread_getattr_np(pthread_self(),&threadAttributes); - #else - pthread_attr_get_np(pthread_self(), &threadAttributes); - #endif - Uptr stackSize; - pthread_attr_getstack(&threadAttributes,(void**)&stackMinAddr,&stackSize); - pthread_attr_destroy(&threadAttributes); - stackMaxAddr = stackMinAddr + stackSize; - stackMinAddr = stackMaxAddr - stackLimit.rlim_cur; - #else - // MacOS uses pthread_getstackaddr_np, and returns a pointer to the maximum address of the stack. - stackMaxAddr = (U8*)pthread_get_stackaddr_np(pthread_self()); - stackMinAddr = stackMaxAddr - stackLimit.rlim_cur; - #endif - - // Include an extra page below the stack's usable address range to distinguish stack overflows from general SIGSEGV. - const Uptr pageSize = sysconf(_SC_PAGESIZE); - stackMinAddr -= pageSize; - } - } - - THREAD_LOCAL sigjmp_buf signalReturnEnv; - THREAD_LOCAL HardwareTrapType signalType = HardwareTrapType::none; - THREAD_LOCAL CallStack* signalCallStack = nullptr; - THREAD_LOCAL Uptr* signalOperand = nullptr; - THREAD_LOCAL bool isReentrantSignal = false; - THREAD_LOCAL bool isCatchingSignals = false; - thread_local std::exception_ptr thrown_exception; - - void signalHandler(int signalNumber,siginfo_t* signalInfo,void*) - { - if(isReentrantSignal) { Errors::fatal("reentrant signal handler"); } - isReentrantSignal = true; - - // Derive the exception cause the from signal that was received. - switch(signalNumber) - { - case SIGFPE: - if(signalInfo->si_code != FPE_INTDIV && signalInfo->si_code != FPE_INTOVF) { Errors::fatal("unknown SIGFPE code"); } - signalType = HardwareTrapType::intDivideByZeroOrOverflow; - break; - case SIGSEGV: - case SIGBUS: - signalType = signalInfo->si_addr >= stackMinAddr && signalInfo->si_addr < stackMaxAddr - ? HardwareTrapType::stackOverflow - : HardwareTrapType::accessViolation; - *signalOperand = reinterpret_cast(signalInfo->si_addr); - break; - default: - Errors::fatalf("unknown signal number: %i",signalNumber); - break; - }; - - // Capture the execution context, omitting this function and the function that called it, - // so the top of the callstack is the function that triggered the signal. - *signalCallStack = captureCallStack(2); - - // If the signal occurred outside of a catchHardwareTraps call, just treat it as a fatal error. - if(!isCatchingSignals) - { - switch(signalNumber) - { - case SIGFPE: Errors::fatalf("unhandled SIGFPE\n"); - case SIGSEGV: Errors::fatalf("unhandled SIGSEGV\n"); - case SIGBUS: Errors::fatalf("unhandled SIGBUS\n"); - default: Errors::unreachable(); - }; - } - - // Jump back to the setjmp in catchHardwareTraps. - siglongjmp(signalReturnEnv,1); - } - - THREAD_LOCAL bool hasInitializedSignalHandlers = false; - - void initSignals() - { - if(!hasInitializedSignalHandlers) - { - hasInitializedSignalHandlers = true; - - // Set a signal handler for the signals we want to intercept. - struct sigaction signalAction; - signalAction.sa_sigaction = signalHandler; - sigemptyset(&signalAction.sa_mask); - signalAction.sa_flags = SA_SIGINFO | SA_ONSTACK; - sigaction(SIGSEGV,&signalAction,nullptr); - sigaction(SIGBUS,&signalAction,nullptr); - sigaction(SIGFPE,&signalAction,nullptr); - } - } - - HardwareTrapType catchHardwareTraps( - CallStack& outTrapCallStack, - Uptr& outTrapOperand, - const std::function& thunk - ) - { - initThread(); - initSignals(); - - jmp_buf oldSignalReturnEnv; - memcpy(&oldSignalReturnEnv,&signalReturnEnv,sizeof(jmp_buf)); - const bool oldIsCatchingSignals = isCatchingSignals; - thrown_exception = nullptr; - - // Use setjmp to allow signals to jump back to this point. - bool isReturningFromSignalHandler = sigsetjmp(signalReturnEnv,1); - if(!isReturningFromSignalHandler) - { - signalType = HardwareTrapType::none; - signalCallStack = &outTrapCallStack; - signalOperand = &outTrapOperand; - isCatchingSignals = true; - - // Call the thunk. - thunk(); - } - - // Reset the signal state. - memcpy(&signalReturnEnv,&oldSignalReturnEnv,sizeof(jmp_buf)); - isCatchingSignals = oldIsCatchingSignals; - isReentrantSignal = false; - signalCallStack = nullptr; - signalOperand = nullptr; - - if(thrown_exception) - std::rethrow_exception(thrown_exception); - - return signalType; - } - - void immediately_exit(std::exception_ptr except) { - thrown_exception = except; - siglongjmp(signalReturnEnv,1); - } - - CallStack captureCallStack(Uptr numOmittedFramesFromTop) - { - #if 0 - // Unwind the callstack. - enum { maxCallStackSize = signalStackNumBytes / sizeof(void*) / 8 }; - void* callstackAddresses[maxCallStackSize]; - auto numCallStackEntries = backtrace(callstackAddresses,maxCallStackSize); - - // Copy the return pointers into the stack frames of the resulting ExecutionContext. - // Skip the first numOmittedFramesFromTop+1 frames, which correspond to this function - // and others that the caller would like to omit. - CallStack result; - for(Iptr index = numOmittedFramesFromTop + 1;index < numCallStackEntries;++index) - { - result.stackFrames.push_back({(Uptr)callstackAddresses[index]}); - } - return result; - #else - return CallStack(); - #endif - } - - U64 getMonotonicClock() - { - #ifdef __APPLE__ - timeval timeVal; - gettimeofday(&timeVal, nullptr); - return U64(timeVal.tv_sec) * 1000000 + U64(timeVal.tv_usec); - #else - timespec monotonicClock; - clock_gettime(CLOCK_MONOTONIC,&monotonicClock); - return U64(monotonicClock.tv_sec) * 1000000 + U64(monotonicClock.tv_nsec) / 1000; - #endif - } - - struct Mutex - { - pthread_mutex_t pthreadMutex; - }; - - Mutex* createMutex() - { - auto mutex = new Mutex(); - errorUnless(!pthread_mutex_init(&mutex->pthreadMutex,nullptr)); - return mutex; - } - - void destroyMutex(Mutex* mutex) - { - errorUnless(!pthread_mutex_destroy(&mutex->pthreadMutex)); - delete mutex; - } - - void lockMutex(Mutex* mutex) - { - errorUnless(!pthread_mutex_lock(&mutex->pthreadMutex)); - } - - void unlockMutex(Mutex* mutex) - { - errorUnless(!pthread_mutex_unlock(&mutex->pthreadMutex)); - } - - struct Event - { - pthread_cond_t conditionVariable; - pthread_mutex_t mutex; - }; - - Event* createEvent() - { - auto event = new Event(); - - pthread_condattr_t conditionVariableAttr; - errorUnless(!pthread_condattr_init(&conditionVariableAttr)); - - // Set the condition variable to use the monotonic clock for wait timeouts. - #ifndef __APPLE__ - errorUnless(!pthread_condattr_setclock(&conditionVariableAttr,CLOCK_MONOTONIC)); - #endif - - errorUnless(!pthread_cond_init(&event->conditionVariable,nullptr)); - errorUnless(!pthread_mutex_init(&event->mutex,nullptr)); - - errorUnless(!pthread_condattr_destroy(&conditionVariableAttr)); - - return event; - } - - void destroyEvent(Event* event) - { - pthread_cond_destroy(&event->conditionVariable); - errorUnless(!pthread_mutex_destroy(&event->mutex)); - delete event; - } - - bool waitForEvent(Event* event,U64 untilTime) - { - errorUnless(!pthread_mutex_lock(&event->mutex)); - - int result; - if(untilTime == UINT64_MAX) - { - result = pthread_cond_wait(&event->conditionVariable,&event->mutex); - } - else - { - timespec untilTimeSpec; - untilTimeSpec.tv_sec = untilTime / 1000000; - untilTimeSpec.tv_nsec = (untilTime % 1000000) * 1000; - - result = pthread_cond_timedwait(&event->conditionVariable,&event->mutex,&untilTimeSpec); - } - - errorUnless(!pthread_mutex_unlock(&event->mutex)); - - if(result == ETIMEDOUT) - { - return false; - } - else - { - errorUnless(!result); - return true; - } - } - - void signalEvent(Event* event) - { - errorUnless(!pthread_cond_signal(&event->conditionVariable)); - } -} - -#endif diff --git a/libraries/wasm-jit/Source/Platform/Windows.cpp b/libraries/wasm-jit/Source/Platform/Windows.cpp deleted file mode 100644 index 2c34c613b64..00000000000 --- a/libraries/wasm-jit/Source/Platform/Windows.cpp +++ /dev/null @@ -1,368 +0,0 @@ -#if _WIN32 - -#include "Inline/BasicTypes.h" -#include "Inline/Errors.h" -#include "Platform.h" - -#include -#include -#include -#include - -#undef min - -namespace Platform -{ - static Uptr internalGetPreferredVirtualPageSizeLog2() - { - SYSTEM_INFO systemInfo; - GetSystemInfo(&systemInfo); - Uptr preferredVirtualPageSize = systemInfo.dwPageSize; - // Verify our assumption that the virtual page size is a power of two. - errorUnless(!(preferredVirtualPageSize & (preferredVirtualPageSize - 1))); - return floorLogTwo(preferredVirtualPageSize); - } - Uptr getPageSizeLog2() - { - static Uptr preferredVirtualPageSizeLog2 = internalGetPreferredVirtualPageSizeLog2(); - return preferredVirtualPageSizeLog2; - } - - U32 memoryAccessAsWin32Flag(MemoryAccess access) - { - switch(access) - { - default: - case MemoryAccess::None: return PAGE_NOACCESS; - case MemoryAccess::ReadOnly: return PAGE_READONLY; - case MemoryAccess::ReadWrite: return PAGE_READWRITE; - case MemoryAccess::Execute: return PAGE_EXECUTE_READ; - case MemoryAccess::ReadWriteExecute: return PAGE_EXECUTE_READWRITE; - } - } - - static bool isPageAligned(U8* address) - { - const Uptr addressBits = reinterpret_cast(address); - return (addressBits & ((1ull << getPageSizeLog2()) - 1)) == 0; - } - - U8* allocateVirtualPages(Uptr numPages) - { - Uptr numBytes = numPages << getPageSizeLog2(); - auto result = VirtualAlloc(nullptr,numBytes,MEM_RESERVE,PAGE_NOACCESS); - if(result == NULL) - { - return nullptr; - } - return (U8*)result; - } - - bool commitVirtualPages(U8* baseVirtualAddress,Uptr numPages,MemoryAccess access) - { - errorUnless(isPageAligned(baseVirtualAddress)); - return baseVirtualAddress == VirtualAlloc(baseVirtualAddress,numPages << getPageSizeLog2(),MEM_COMMIT,memoryAccessAsWin32Flag(access)); - } - - bool setVirtualPageAccess(U8* baseVirtualAddress,Uptr numPages,MemoryAccess access) - { - errorUnless(isPageAligned(baseVirtualAddress)); - DWORD oldProtection = 0; - return VirtualProtect(baseVirtualAddress,numPages << getPageSizeLog2(),memoryAccessAsWin32Flag(access),&oldProtection) != 0; - } - - void decommitVirtualPages(U8* baseVirtualAddress,Uptr numPages) - { - errorUnless(isPageAligned(baseVirtualAddress)); - auto result = VirtualFree(baseVirtualAddress,numPages << getPageSizeLog2(),MEM_DECOMMIT); - if(baseVirtualAddress && !result) { Errors::fatal("VirtualFree(MEM_DECOMMIT) failed"); } - } - - void freeVirtualPages(U8* baseVirtualAddress,Uptr numPages) - { - errorUnless(isPageAligned(baseVirtualAddress)); - auto result = VirtualFree(baseVirtualAddress,0/*numPages << getPageSizeLog2()*/,MEM_RELEASE); - if(baseVirtualAddress && !result) { Errors::fatal("VirtualFree(MEM_RELEASE) failed"); } - } - - // The interface to the DbgHelp DLL - struct DbgHelp - { - typedef BOOL (WINAPI* SymFromAddr)(HANDLE,U64,U64*,SYMBOL_INFO*); - SymFromAddr symFromAddr; - DbgHelp() - { - HMODULE dbgHelpModule = ::LoadLibraryA("Dbghelp.dll"); - if(dbgHelpModule) - { - symFromAddr = (SymFromAddr)::GetProcAddress(dbgHelpModule,"SymFromAddr"); - - // Initialize the debug symbol lookup. - typedef BOOL (WINAPI* SymInitialize)(HANDLE,PCTSTR,BOOL); - SymInitialize symInitialize = (SymInitialize)::GetProcAddress(dbgHelpModule,"SymInitialize"); - if(symInitialize) - { - symInitialize(GetCurrentProcess(),nullptr,TRUE); - } - } - } - }; - DbgHelp* dbgHelp = nullptr; - - bool describeInstructionPointer(Uptr ip,std::string& outDescription) - { - // Initialize DbgHelp. - if(!dbgHelp) { dbgHelp = new DbgHelp(); } - - // Allocate up a SYMBOL_INFO struct to receive information about the symbol for this instruction pointer. - const Uptr maxSymbolNameChars = 256; - const Uptr symbolAllocationSize = sizeof(SYMBOL_INFO) + sizeof(TCHAR) * (maxSymbolNameChars - 1); - SYMBOL_INFO* symbolInfo = (SYMBOL_INFO*)alloca(symbolAllocationSize); - ZeroMemory(symbolInfo,symbolAllocationSize); - symbolInfo->SizeOfStruct = sizeof(SYMBOL_INFO); - symbolInfo->MaxNameLen = maxSymbolNameChars; - - // Call DbgHelp::SymFromAddr to try to find any debug symbol containing this address. - if(!dbgHelp->symFromAddr(GetCurrentProcess(),ip,nullptr,symbolInfo)) { return false; } - else - { - outDescription = symbolInfo->Name; - return true; - } - } - - #if defined(_WIN32) && defined(_AMD64_) - void registerSEHUnwindInfo(Uptr imageLoadAddress,Uptr pdataAddress,Uptr pdataNumBytes) - { - const U32 numFunctions = (U32)(pdataNumBytes / sizeof(RUNTIME_FUNCTION)); - - // Register our manually fixed up copy of the function table. - if(!RtlAddFunctionTable(reinterpret_cast(pdataAddress),numFunctions,imageLoadAddress)) - { - Errors::fatal("RtlAddFunctionTable failed"); - } - } - void deregisterSEHUnwindInfo(Uptr pdataAddress) - { - auto functionTable = reinterpret_cast(pdataAddress); - RtlDeleteFunctionTable(functionTable); - delete [] functionTable; - } - #endif - - CallStack unwindStack(const CONTEXT& immutableContext) - { - // Make a mutable copy of the context. - CONTEXT context; - memcpy(&context,&immutableContext,sizeof(CONTEXT)); - - // Unwind the stack until there isn't a valid instruction pointer, which signals we've reached the base. - CallStack callStack; - #ifdef _WIN64 - while(context.Rip) - { - callStack.stackFrames.push_back({context.Rip}); - - // Look up the SEH unwind information for this function. - U64 imageBase; - auto runtimeFunction = RtlLookupFunctionEntry(context.Rip,&imageBase,nullptr); - if(!runtimeFunction) - { - // Leaf functions that don't touch Rsp may not have unwind information. - context.Rip = *(U64*)context.Rsp; - context.Rsp += 8; - } - else - { - // Use the SEH information to unwind to the next stack frame. - void* handlerData; - U64 establisherFrame; - RtlVirtualUnwind( - UNW_FLAG_NHANDLER, - imageBase, - context.Rip, - runtimeFunction, - &context, - &handlerData, - &establisherFrame, - nullptr - ); - } - } - #endif - - return callStack; - } - - CallStack captureCallStack(Uptr numOmittedFramesFromTop) - { - // Capture the current processor state. - CONTEXT context; - RtlCaptureContext(&context); - - // Unwind the stack. - auto result = unwindStack(context); - - // Remote the requested number of omitted frames, +1 for this function. - const Uptr numOmittedFrames = std::min(result.stackFrames.size(),numOmittedFramesFromTop + 1); - result.stackFrames.erase(result.stackFrames.begin(),result.stackFrames.begin() + numOmittedFrames); - - return result; - } - - THREAD_LOCAL bool isReentrantException = false; - LONG CALLBACK sehFilterFunction(EXCEPTION_POINTERS* exceptionPointers,HardwareTrapType& outType,Uptr& outTrapOperand,CallStack& outCallStack) - { - if(isReentrantException) { Errors::fatal("reentrant exception"); } - else - { - // Decide how to handle this exception code. - switch(exceptionPointers->ExceptionRecord->ExceptionCode) - { - case EXCEPTION_ACCESS_VIOLATION: - outType = HardwareTrapType::accessViolation; - outTrapOperand = exceptionPointers->ExceptionRecord->ExceptionInformation[1]; - break; - case EXCEPTION_STACK_OVERFLOW: outType = HardwareTrapType::stackOverflow; break; - case STATUS_INTEGER_DIVIDE_BY_ZERO: outType = HardwareTrapType::intDivideByZeroOrOverflow; break; - case STATUS_INTEGER_OVERFLOW: outType = HardwareTrapType::intDivideByZeroOrOverflow; break; - default: return EXCEPTION_CONTINUE_SEARCH; - } - isReentrantException = true; - - // Unwind the stack frames from the context of the exception. - outCallStack = unwindStack(*exceptionPointers->ContextRecord); - - return EXCEPTION_EXECUTE_HANDLER; - } - } - - THREAD_LOCAL bool isThreadInitialized = false; - void initThread() - { - if(!isThreadInitialized) - { - isThreadInitialized = true; - - // Ensure that there's enough space left on the stack in the case of a stack overflow to prepare the stack trace. - ULONG stackOverflowReserveBytes = 32768; - SetThreadStackGuarantee(&stackOverflowReserveBytes); - } - } - - HardwareTrapType catchHardwareTraps( - CallStack& outTrapCallStack, - Uptr& outTrapOperand, - const std::function& thunk - ) - { - initThread(); - - HardwareTrapType result = HardwareTrapType::none; - __try - { - thunk(); - } - __except(sehFilterFunction(GetExceptionInformation(),result,outTrapOperand,outTrapCallStack)) - { - isReentrantException = false; - - // After a stack overflow, the stack will be left in a damaged state. Let the CRT repair it. - if(result == HardwareTrapType::stackOverflow) { _resetstkoflw(); } - } - return result; - } - - U64 getMonotonicClock() - { - LARGE_INTEGER performanceCounter; - LARGE_INTEGER performanceCounterFrequency; - QueryPerformanceCounter(&performanceCounter); - QueryPerformanceFrequency(&performanceCounterFrequency); - - const U64 wavmFrequency = 1000000; - - return performanceCounterFrequency.QuadPart > wavmFrequency - ? performanceCounter.QuadPart / (performanceCounterFrequency.QuadPart / wavmFrequency) - : performanceCounter.QuadPart * (wavmFrequency / performanceCounterFrequency.QuadPart); - } - - struct Mutex - { - CRITICAL_SECTION criticalSection; - }; - - Mutex* createMutex() - { - auto mutex = new Mutex(); - InitializeCriticalSection(&mutex->criticalSection); - return mutex; - } - - void destroyMutex(Mutex* mutex) - { - DeleteCriticalSection(&mutex->criticalSection); - delete mutex; - } - - void lockMutex(Mutex* mutex) - { - EnterCriticalSection(&mutex->criticalSection); - } - - void unlockMutex(Mutex* mutex) - { - LeaveCriticalSection(&mutex->criticalSection); - } - - Event* createEvent() - { - return reinterpret_cast(CreateEvent(nullptr,FALSE,FALSE,nullptr)); - } - - void destroyEvent(Event* event) - { - CloseHandle(reinterpret_cast(event)); - } - - bool waitForEvent(Event* event,U64 untilTime) - { - U64 currentTime = getMonotonicClock(); - const U64 startProcessTime = currentTime; - while(true) - { - const U64 timeoutMicroseconds = currentTime > untilTime ? 0 : (untilTime - currentTime); - const U64 timeoutMilliseconds64 = timeoutMicroseconds / 1000; - const U32 timeoutMilliseconds32 = - timeoutMilliseconds64 > UINT32_MAX - ? (UINT32_MAX - 1) - : U32(timeoutMilliseconds64); - - const U32 waitResult = WaitForSingleObject(reinterpret_cast(event),timeoutMilliseconds32); - if(waitResult != WAIT_TIMEOUT) - { - errorUnless(waitResult == WAIT_OBJECT_0); - return true; - } - else - { - currentTime = getMonotonicClock(); - if(currentTime >= untilTime) - { - return false; - } - } - }; - } - - void signalEvent(Event* event) - { - errorUnless(SetEvent(reinterpret_cast(event))); - } - - void immediately_exit(std::exception_ptr except) { - std::rethrow_exception(except); - } -} - -#endif diff --git a/libraries/wasm-jit/Source/Runtime/CMakeLists.txt b/libraries/wasm-jit/Source/Runtime/CMakeLists.txt index 907dbf37871..4fb6d2e234e 100644 --- a/libraries/wasm-jit/Source/Runtime/CMakeLists.txt +++ b/libraries/wasm-jit/Source/Runtime/CMakeLists.txt @@ -4,7 +4,6 @@ set(Sources LLVMJIT.h ObjectGC.cpp RuntimePrivate.h - WAVMIntrinsics.cpp ) set(PublicHeaders @@ -21,7 +20,7 @@ add_definitions(-DRUNTIME_API=DLL_EXPORT) target_include_directories( Runtime PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../../../chain/include ) # Link against the LLVM libraries -target_link_libraries(Runtime Platform Logging IR) +target_link_libraries(Runtime Logging IR) install(TARGETS Runtime LIBRARY DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR} diff --git a/libraries/wasm-jit/Source/Runtime/Intrinsics.cpp b/libraries/wasm-jit/Source/Runtime/Intrinsics.cpp index 11f5dc3a3f1..32cd1f7e940 100644 --- a/libraries/wasm-jit/Source/Runtime/Intrinsics.cpp +++ b/libraries/wasm-jit/Source/Runtime/Intrinsics.cpp @@ -11,9 +11,8 @@ namespace Intrinsics struct Singleton { std::map functionMap; - Platform::Mutex* mutex; - Singleton(): mutex(Platform::createMutex()) {} + Singleton() {} Singleton(const Singleton&) = delete; static Singleton& get() @@ -35,14 +34,12 @@ namespace Intrinsics : name(inName) { function = new Runtime::FunctionInstance(nullptr,type,nativeFunction); - Platform::Lock lock(Singleton::get().mutex); Singleton::get().functionMap[getDecoratedName(inName,type)] = this; } Function::~Function() { { - Platform::Lock Lock(Singleton::get().mutex); Singleton::get().functionMap.erase(Singleton::get().functionMap.find(getDecoratedName(name,function->type))); } delete function; @@ -51,7 +48,6 @@ namespace Intrinsics Runtime::ObjectInstance* find(const std::string& name,const IR::ObjectType& type) { std::string decoratedName = getDecoratedName(name,type); - Platform::Lock Lock(Singleton::get().mutex); Runtime::ObjectInstance* result = nullptr; switch(type.kind) { @@ -84,7 +80,6 @@ namespace Intrinsics std::vector getAllIntrinsicObjects() { - Platform::Lock lock(Singleton::get().mutex); std::vector result; for(auto mapIt : Singleton::get().functionMap) { result.push_back(mapIt.second->function); } return result; diff --git a/libraries/wasm-jit/Source/Runtime/Linker.cpp b/libraries/wasm-jit/Source/Runtime/Linker.cpp index 0e73cefbdad..a0aba49e084 100644 --- a/libraries/wasm-jit/Source/Runtime/Linker.cpp +++ b/libraries/wasm-jit/Source/Runtime/Linker.cpp @@ -80,4 +80,4 @@ namespace Runtime linkResult.success = linkResult.missingImports.size() == 0; return linkResult; } -} \ No newline at end of file +} diff --git a/libraries/wasm-jit/Source/Runtime/RuntimePrivate.h b/libraries/wasm-jit/Source/Runtime/RuntimePrivate.h index c180b31330f..590a6f65aab 100644 --- a/libraries/wasm-jit/Source/Runtime/RuntimePrivate.h +++ b/libraries/wasm-jit/Source/Runtime/RuntimePrivate.h @@ -142,20 +142,6 @@ namespace Runtime ~ModuleInstance() override; }; - // Initializes global state used by the WAVM intrinsics. - void initWAVMIntrinsics(); - - // Checks whether an address is owned by a table or memory. - bool isAddressOwnedByTable(U8* address); - bool isAddressOwnedByMemory(U8* address); - - // Allocates virtual pages with alignBytes of padding, and returns an aligned base address. - // The unaligned allocation address and size are written to outUnalignedBaseAddress and outUnalignedNumPlatformPages. - U8* allocateVirtualPagesAligned(Uptr numBytes,Uptr alignmentBytes,U8*& outUnalignedBaseAddress,Uptr& outUnalignedNumPlatformPages); - - // Turns a hardware trap that occurred in WASM code into a runtime exception or fatal error. - [[noreturn]] void handleHardwareTrap(Platform::HardwareTrapType trapType,Platform::CallStack&& trapCallStack,Uptr trapOperand); - // Adds GC roots from WASM threads to the provided array. void getThreadGCRoots(std::vector& outGCRoots); } diff --git a/libraries/wasm-jit/Source/Runtime/WAVMIntrinsics.cpp b/libraries/wasm-jit/Source/Runtime/WAVMIntrinsics.cpp deleted file mode 100644 index 9e38eeebea0..00000000000 --- a/libraries/wasm-jit/Source/Runtime/WAVMIntrinsics.cpp +++ /dev/null @@ -1,222 +0,0 @@ -#include "Inline/BasicTypes.h" -#include "Inline/Floats.h" -#include "Logging/Logging.h" -#include "Intrinsics.h" -#include "RuntimePrivate.h" - -#include - -namespace Runtime -{ - static void causeIntrensicException(Exception::Cause cause) { - try { - Platform::immediately_exit(std::make_exception_ptr(Exception{cause, std::vector()})); - } - catch (...) { - Platform::immediately_exit(std::current_exception()); - } - __builtin_unreachable(); - } - - template - Float quietNaN(Float value) - { - Floats::FloatComponents components; - components.value = value; - components.bits.significand |= typename Floats::FloatComponents::Bits(1) << (Floats::FloatComponents::numSignificandBits - 1); - return components.value; - } - - template - Float floatMin(Float left,Float right) - { - // If either operand is a NaN, convert it to a quiet NaN and return it. - if(left != left) { return quietNaN(left); } - else if(right != right) { return quietNaN(right); } - // If either operand is less than the other, return it. - else if(left < right) { return left; } - else if(right < left) { return right; } - else - { - // Finally, if the operands are apparently equal, compare their integer values to distinguish -0.0 from +0.0 - Floats::FloatComponents leftComponents; - leftComponents.value = left; - Floats::FloatComponents rightComponents; - rightComponents.value = right; - return leftComponents.bitcastInt < rightComponents.bitcastInt ? right : left; - } - } - - template - Float floatMax(Float left,Float right) - { - // If either operand is a NaN, convert it to a quiet NaN and return it. - if(left != left) { return quietNaN(left); } - else if(right != right) { return quietNaN(right); } - // If either operand is less than the other, return it. - else if(left > right) { return left; } - else if(right > left) { return right; } - else - { - // Finally, if the operands are apparently equal, compare their integer values to distinguish -0.0 from +0.0 - Floats::FloatComponents leftComponents; - leftComponents.value = left; - Floats::FloatComponents rightComponents; - rightComponents.value = right; - return leftComponents.bitcastInt > rightComponents.bitcastInt ? right : left; - } - } - - template - Float floatCeil(Float value) - { - if(value != value) { return quietNaN(value); } - else { return ceil(value); } - } - - template - Float floatFloor(Float value) - { - if(value != value) { return quietNaN(value); } - else { return floor(value); } - } - - template - Float floatTrunc(Float value) - { - if(value != value) { return quietNaN(value); } - else { return trunc(value); } - } - - template - Float floatNearest(Float value) - { - if(value != value) { return quietNaN(value); } - else { return nearbyint(value); } - } - - DEFINE_INTRINSIC_FUNCTION2(wavmIntrinsics,floatMin,floatMin,f32,f32,left,f32,right) { return floatMin(left,right); } - DEFINE_INTRINSIC_FUNCTION2(wavmIntrinsics,floatMin,floatMin,f64,f64,left,f64,right) { return floatMin(left,right); } - DEFINE_INTRINSIC_FUNCTION2(wavmIntrinsics,floatMax,floatMax,f32,f32,left,f32,right) { return floatMax(left,right); } - DEFINE_INTRINSIC_FUNCTION2(wavmIntrinsics,floatMax,floatMax,f64,f64,left,f64,right) { return floatMax(left,right); } - - DEFINE_INTRINSIC_FUNCTION1(wavmIntrinsics,floatCeil,floatCeil,f32,f32,value) { return floatCeil(value); } - DEFINE_INTRINSIC_FUNCTION1(wavmIntrinsics,floatCeil,floatCeil,f64,f64,value) { return floatCeil(value); } - DEFINE_INTRINSIC_FUNCTION1(wavmIntrinsics,floatFloor,floatFloor,f32,f32,value) { return floatFloor(value); } - DEFINE_INTRINSIC_FUNCTION1(wavmIntrinsics,floatFloor,floatFloor,f64,f64,value) { return floatFloor(value); } - DEFINE_INTRINSIC_FUNCTION1(wavmIntrinsics,floatTrunc,floatTrunc,f32,f32,value) { return floatTrunc(value); } - DEFINE_INTRINSIC_FUNCTION1(wavmIntrinsics,floatTrunc,floatTrunc,f64,f64,value) { return floatTrunc(value); } - DEFINE_INTRINSIC_FUNCTION1(wavmIntrinsics,floatNearest,floatNearest,f32,f32,value) { return floatNearest(value); } - DEFINE_INTRINSIC_FUNCTION1(wavmIntrinsics,floatNearest,floatNearest,f64,f64,value) { return floatNearest(value); } - - template - Dest floatToInt(Source sourceValue,Source minValue,Source maxValue) - { - if(sourceValue != sourceValue) - { - causeIntrensicException(Exception::Cause::invalidFloatOperation); - } - else if(sourceValue >= maxValue || (isMinInclusive ? sourceValue <= minValue : sourceValue < minValue)) - { - causeIntrensicException(Exception::Cause::integerDivideByZeroOrIntegerOverflow); - } - return (Dest)sourceValue; - } - - DEFINE_INTRINSIC_FUNCTION1(wavmIntrinsics,floatToSignedInt,floatToSignedInt,i32,f32,source) { return floatToInt(source,(F32)INT32_MIN,-(F32)INT32_MIN); } - DEFINE_INTRINSIC_FUNCTION1(wavmIntrinsics,floatToSignedInt,floatToSignedInt,i32,f64,source) { return floatToInt(source,(F64)INT32_MIN,-(F64)INT32_MIN); } - DEFINE_INTRINSIC_FUNCTION1(wavmIntrinsics,floatToSignedInt,floatToSignedInt,i64,f32,source) { return floatToInt(source,(F32)INT64_MIN,-(F32)INT64_MIN); } - DEFINE_INTRINSIC_FUNCTION1(wavmIntrinsics,floatToSignedInt,floatToSignedInt,i64,f64,source) { return floatToInt(source,(F64)INT64_MIN,-(F64)INT64_MIN); } - - DEFINE_INTRINSIC_FUNCTION1(wavmIntrinsics,floatToUnsignedInt,floatToUnsignedInt,i32,f32,source) { return floatToInt(source,-1.0f,-2.0f * INT32_MIN); } - DEFINE_INTRINSIC_FUNCTION1(wavmIntrinsics,floatToUnsignedInt,floatToUnsignedInt,i32,f64,source) { return floatToInt(source,-1.0,-2.0 * INT32_MIN); } - DEFINE_INTRINSIC_FUNCTION1(wavmIntrinsics,floatToUnsignedInt,floatToUnsignedInt,i64,f32,source) { return floatToInt(source,-1.0f,-2.0f * INT64_MIN); } - DEFINE_INTRINSIC_FUNCTION1(wavmIntrinsics,floatToUnsignedInt,floatToUnsignedInt,i64,f64,source) { return floatToInt(source,-1.0,-2.0 * INT64_MIN); } - - DEFINE_INTRINSIC_FUNCTION0(wavmIntrinsics,divideByZeroOrIntegerOverflowTrap,divideByZeroOrIntegerOverflowTrap,none) - { - causeIntrensicException(Exception::Cause::integerDivideByZeroOrIntegerOverflow); - } - - DEFINE_INTRINSIC_FUNCTION0(wavmIntrinsics,unreachableTrap,unreachableTrap,none) - { - causeIntrensicException(Exception::Cause::reachedUnreachable); - } - - DEFINE_INTRINSIC_FUNCTION0(wavmIntrinsics,accessViolationTrap,accessViolationTrap,none) - { - causeIntrensicException(Exception::Cause::accessViolation); - } - - DEFINE_INTRINSIC_FUNCTION3(wavmIntrinsics,indirectCallSignatureMismatch,indirectCallSignatureMismatch,none,i32,index,i64,expectedSignatureBits,i64,tableBits) - { - try { - TableInstance* table = reinterpret_cast(tableBits); - void* elementValue = table->baseAddress[index].value; - const FunctionType* actualSignature = table->baseAddress[index].type; - const FunctionType* expectedSignature = reinterpret_cast((Uptr)expectedSignatureBits); - std::string ipDescription = ""; - LLVMJIT::describeInstructionPointer(reinterpret_cast(elementValue),ipDescription); - Log::printf(Log::Category::debug,"call_indirect signature mismatch: expected %s at index %u but got %s (%s)\n", - asString(expectedSignature).c_str(), - index, - actualSignature ? asString(actualSignature).c_str() : "nullptr", - ipDescription.c_str() - ); - causeIntrensicException(elementValue == nullptr ? Exception::Cause::undefinedTableElement : Exception::Cause::indirectCallSignatureMismatch); - } - catch (...) { - Platform::immediately_exit(std::current_exception()); - } - } - - DEFINE_INTRINSIC_FUNCTION0(wavmIntrinsics,indirectCallIndexOutOfBounds,indirectCallIndexOutOfBounds,none) - { - causeIntrensicException(Exception::Cause::undefinedTableElement); - } - - DEFINE_INTRINSIC_FUNCTION2(wavmIntrinsics,_growMemory,growMemory,i32,i32,deltaPages,i64,memoryBits) - { - MemoryInstance* memory = reinterpret_cast(memoryBits); - if(!memory) - causeIntrensicException(Exception::Cause::outOfMemory); - const Iptr numPreviousMemoryPages = growMemory(memory,(Uptr)deltaPages); - if(numPreviousMemoryPages + (Uptr)deltaPages > IR::maxMemoryPages) { return -1; } - else { return (I32)numPreviousMemoryPages; } - } - - DEFINE_INTRINSIC_FUNCTION1(wavmIntrinsics,_currentMemory,currentMemory,i32,i64,memoryBits) - { - MemoryInstance* memory = reinterpret_cast(memoryBits); - if(!memory) - causeIntrensicException(Exception::Cause::outOfMemory); - Uptr numMemoryPages = getMemoryNumPages(memory); - if(numMemoryPages > UINT32_MAX) { numMemoryPages = UINT32_MAX; } - return (U32)numMemoryPages; - } - - THREAD_LOCAL Uptr indentLevel = 0; - - DEFINE_INTRINSIC_FUNCTION1(wavmIntrinsics,debugEnterFunction,debugEnterFunction,none,i64,functionInstanceBits) - { - FunctionInstance* function = reinterpret_cast(functionInstanceBits); - Log::printf(Log::Category::debug,"ENTER: %s\n",function->debugName.c_str()); - ++indentLevel; - } - - DEFINE_INTRINSIC_FUNCTION1(wavmIntrinsics,debugExitFunction,debugExitFunction,none,i64,functionInstanceBits) - { - FunctionInstance* function = reinterpret_cast(functionInstanceBits); - --indentLevel; - Log::printf(Log::Category::debug,"EXIT: %s\n",function->debugName.c_str()); - } - - DEFINE_INTRINSIC_FUNCTION0(wavmIntrinsics,debugBreak,debugBreak,none) - { - Log::printf(Log::Category::debug,"================== wavmIntrinsics.debugBreak\n"); - } - - void initWAVMIntrinsics() - { - } -} diff --git a/libraries/wasm-jit/Source/WASM/WASMSerialization.cpp b/libraries/wasm-jit/Source/WASM/WASMSerialization.cpp index f70fdbbb164..95c678cee7c 100644 --- a/libraries/wasm-jit/Source/WASM/WASMSerialization.cpp +++ b/libraries/wasm-jit/Source/WASM/WASMSerialization.cpp @@ -179,7 +179,9 @@ namespace WASM { using namespace IR; using namespace Serialization; - + + bool check_limits = true; + enum { magicNumber=0x6d736100, // "\0asm" @@ -490,9 +492,9 @@ namespace WASM serializeVarUInt32(bodyStream,numLocalSets); constexpr size_t max_size = eosio::chain::wasm_constraints::maximum_code_size; - if (numBodyBytes >= max_size) + if (numBodyBytes >= max_size && WASM::check_limits) throw FatalSerializationException(std::string("Function body too large")); - if (numLocalSets >= 1024) + if (numLocalSets >= 1024 && WASM::check_limits) throw FatalSerializationException(std::string("too many local sets")); size_t locals_accum = 0; @@ -501,7 +503,7 @@ namespace WASM LocalSet localSet; serialize(bodyStream,localSet); locals_accum += localSet.num*4; - if( locals_accum > eosio::chain::wasm_constraints::maximum_func_local_bytes ) + if( locals_accum > eosio::chain::wasm_constraints::maximum_func_local_bytes && WASM::check_limits ) throw FatalSerializationException( "too many locals" ); for(Uptr index = 0;index < localSet.num;++index) { functionDef.nonParameterLocalTypes.push_back(localSet.type); } @@ -593,7 +595,7 @@ namespace WASM throw FatalSerializationException("invalid import function type index"); } module.functions.imports.push_back({{functionTypeIndex},std::move(moduleName),std::move(exportName)}); - if (module.functions.imports.size() >= max_size) + if (module.functions.imports.size() >= max_size && WASM::check_limits) throw FatalSerializationException(std::string("Too many function imports")); break; } @@ -602,7 +604,7 @@ namespace WASM TableType tableType; serialize(sectionStream,tableType); module.tables.imports.push_back({tableType,std::move(moduleName),std::move(exportName)}); - if (module.functions.imports.size() >= max_size) + if (module.functions.imports.size() >= max_size && WASM::check_limits) throw FatalSerializationException(std::string("Too many table imports")); break; } @@ -611,7 +613,7 @@ namespace WASM MemoryType memoryType; serialize(sectionStream,memoryType); module.memories.imports.push_back({memoryType,std::move(moduleName),std::move(exportName)}); - if (module.functions.imports.size() >= max_size) + if (module.functions.imports.size() >= max_size && WASM::check_limits) throw FatalSerializationException(std::string("Too many memory imports")); break; } @@ -620,7 +622,7 @@ namespace WASM GlobalType globalType; serialize(sectionStream,globalType); module.globals.imports.push_back({globalType,std::move(moduleName),std::move(exportName)}); - if (module.functions.imports.size() >= max_size) + if (module.functions.imports.size() >= max_size && WASM::check_limits) throw FatalSerializationException(std::string("Too many global imports")); break; } @@ -679,7 +681,7 @@ namespace WASM // try to get a serialization exception before making a huge allocation for malformed input. module.functions.defs.clear(); constexpr size_t max_size = eosio::chain::wasm_constraints::maximum_section_elements; - if ( numFunctions >= max_size ) + if ( numFunctions >= max_size && WASM::check_limits ) throw FatalSerializationException(std::string("Too many function defs")); for(Uptr functionIndex = 0;functionIndex < numFunctions;++functionIndex) { diff --git a/libraries/wasm-jit/Source/WAST/ParseNumbers.cpp b/libraries/wasm-jit/Source/WAST/ParseNumbers.cpp index 4e63586db95..2e41b94486e 100644 --- a/libraries/wasm-jit/Source/WAST/ParseNumbers.cpp +++ b/libraries/wasm-jit/Source/WAST/ParseNumbers.cpp @@ -6,6 +6,7 @@ #include #include +#include // Include the David Gay's dtoa code. // #define strtod and dtoa to avoid conflicting with the C standard library versions diff --git a/package.json b/package.json new file mode 100644 index 00000000000..69e5acdcc96 --- /dev/null +++ b/package.json @@ -0,0 +1,12 @@ +{ + "name": "eosio", + "version": "1.0.0", + "dependencies": { + "eosjs": "^20.0.0", + "ws": "7.2.0", + "commander": "4.0.1", + "zlib": "1.0.5", + "node-fetch": "2.6.0", + "util": "^0.12.3" + } +} diff --git a/pipeline.jsonc b/pipeline.jsonc index b809b704ff3..807760cc7f9 100644 --- a/pipeline.jsonc +++ b/pipeline.jsonc @@ -6,8 +6,6 @@ "^/coverage/", "^/.git/", "^/libraries/eos-vm/", - "^/libraries/fc/", - "^/libraries/wabt/", "^/libraries/wasm-jit/", "^/node_modules/", "^/unittests/" @@ -41,11 +39,8 @@ "test": [ { - "tag": "v2.0.0-rc2" - }, - { - "tag": "v2.0.0-rc3" + "commit": "16074742f8cfd481b029073e6f01bb920a1bad38" } ] } -} \ No newline at end of file +} diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 5f196cc8c4f..8c9fd0d2403 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -10,16 +10,19 @@ add_subdirectory(history_plugin) add_subdirectory(history_api_plugin) add_subdirectory(state_history_plugin) add_subdirectory(trace_api_plugin) - +add_subdirectory(signature_provider_plugin) +add_subdirectory(resource_monitor_plugin) add_subdirectory(wallet_plugin) add_subdirectory(wallet_api_plugin) add_subdirectory(txn_test_gen_plugin) add_subdirectory(db_size_api_plugin) -#add_subdirectory(faucet_testnet_plugin) -add_subdirectory(mongo_db_plugin) add_subdirectory(login_plugin) add_subdirectory(test_control_plugin) add_subdirectory(test_control_api_plugin) +option(ENABLE_BLOCK_VAULT_CLIENT "Enable blockvault_client_plugin" ON) +if (ENABLE_BLOCK_VAULT_CLIENT) + add_subdirectory(blockvault_client_plugin) +endif() # Forward variables to top level so packaging picks them up set(CPACK_DEBIAN_PACKAGE_DEPENDS ${CPACK_DEBIAN_PACKAGE_DEPENDS} PARENT_SCOPE) diff --git a/plugins/blockvault_client_plugin/CMakeLists.txt b/plugins/blockvault_client_plugin/CMakeLists.txt new file mode 100644 index 00000000000..69556b2cae4 --- /dev/null +++ b/plugins/blockvault_client_plugin/CMakeLists.txt @@ -0,0 +1,46 @@ +find_package(PkgConfig REQUIRED) + +add_library( blockvault_client_plugin + blockvault_client_plugin.cpp) +target_link_libraries( blockvault_client_plugin appbase eosio_chain) +target_include_directories( blockvault_client_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) + + +if (APPLE AND EXISTS "/usr/local/opt/libpq/lib/pkgconfig") + # to find libpq installed by homebrew, it is required to set the environment variable PKG_CONFIG_PATH + if (DEFINED ENV{PKG_CONFIG_PATH}) + set(ENV{PKG_CONFIG_PATH} "/usr/local/opt/libpq/lib/pkgconfig:$ENV{PKG_CONFIG_PATH}") + else() + set(ENV{PKG_CONFIG_PATH} "/usr/local/opt/libpq/lib/pkgconfig") + endif() +endif() + +pkg_check_modules(pq IMPORTED_TARGET libpq>=10.0) +pkg_check_modules(pqxx IMPORTED_TARGET libpqxx>=6.0) + +if (pqxx_FOUND) + + if (APPLE) + ################################################################################################### + # Linking libpqxx from homebrew statically could generate `different visibility settings` warnings. + # As a result, we'll just use link it dynamically for now. + ################################################################################################## + set(pqxx_lib PkgConfig::pqxx ) + else() + add_library(pqxx STATIC IMPORTED) + set_property(TARGET pqxx PROPERTY IMPORTED_LOCATION ${pqxx_LIBDIR}/libpqxx.a) + set_property(TARGET pqxx PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${pqxx_STATIC_INCLUDE_DIRS}) + set(pqxx_lib pqxx) + endif() + + add_library(blockvault postgres_backend.cpp) + target_include_directories(blockvault PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) + target_link_libraries(blockvault PUBLIC ${pqxx_lib} PkgConfig::pq eosio_chain) + + target_link_libraries( blockvault_client_plugin blockvault) + target_compile_definitions(blockvault_client_plugin PRIVATE HAS_PQXX) + + add_subdirectory(tests) +else() + message(WARNING "libpqxx is not found, blockvault_client_plugin will not be functional for the lack of available backend") +endif() diff --git a/plugins/blockvault_client_plugin/README.md b/plugins/blockvault_client_plugin/README.md new file mode 100644 index 00000000000..a41cabb6feb --- /dev/null +++ b/plugins/blockvault_client_plugin/README.md @@ -0,0 +1,59 @@ +# Overview + +Block Vault is a component in an EOSIO network architecture which provides a replicated durable storage with strong consistency guarantees for all the input required by a redundant cluster of nodeos nodes to achieve the guarantees outlined in BlockVault: Failover for Nodeos reproduced here: + +* Guarantee against double-production of blocks +* Guarantee against finality violation +* Guarantee of liveness (ability to make progress as a blockchain) + +By facilitating these guarantees, Block Vault allows nodeos to run in a redundant and/or highly available mode without an additional requirement of coordination between the nodes. Block Vault itself does not implement any coordination of nodeos nodes in a cluster. Thus, it merely guarantees that any such coordination, including faulty coordination leading to multiple active block constructing nodeos nodes will be safe as defined by the above guarantees. + +Currently we use PostgreSQL as our durable storage for block vault, other distributed database may be supported in the future. + +## Compiling the blockvault_client_plugin + +blockvault_client_plugin requires libpq version 10 or above and libpqxx version 6 or above to compile. + +For Mac, you can simply use homebrew to install the required dependencies. +``` +brew install libpq libpqxx +``` + +For Linux, the versions of libpq and libpqxx provided by the system package managers may be not be new enough. We recommend to follow the `install libpq` and `install libpqxx` sections of the corresponding dockerfile in `.cicd/platform/pinned` for your platform to install the dependencies. + + +## Running PostgreSQL for testing purpose + +We recommend to use docker + +``` +docker run -p 5432:5432 -e POSTGRES_PASSWORD=password -d postgres +``` + +## Running PostgreSQL for Production + +We recommend to deploy PostgreSQL with HA (high availability) mode and synchronous replication strategy. + +### Database schema + +This plugin would create two tables `BlockData` and `SnapshotData` if it is not already in the database. The tables are created in the following SQL statement. + +``` +CREATE TABLE IF NOT EXISTS BlockData (watermark_bn bigint, watermark_ts bigint, lib bigint, block_num bigint, block_id bytea UNIQUE, previous_block_id bytea, block oid, block_size bigint); +CREATE TABLE IF NOT EXISTS SnapshotData (watermark_bn bigint, watermark_ts bigint, snapshot oid); +``` + +## Plugin configuration + +To use this plugin, `nodeos` has to be configured as a producer with `--block-vault-backend` option. For example + +``` +nodeos --plugin eosio::producer_plugin --producer-name myproducera --plugin eosio::blockvault_client_plugin --block-vault-backend postgresql://user:password@mycompany.com +``` + +For production environment, we recommend to use `PGPASSWORD` environment variable to configure the password instead of embedding the password in the URI. + +``` +export PGPASSWORD=password +nodeos --plugin eosio::producer_plugin --producer-name myproducera --plugin eosio::blockvault_client_plugin --block-vault-backend postgresql://user@mycompany.com +``` diff --git a/plugins/blockvault_client_plugin/backend.hpp b/plugins/blockvault_client_plugin/backend.hpp new file mode 100644 index 00000000000..6582d16c4d5 --- /dev/null +++ b/plugins/blockvault_client_plugin/backend.hpp @@ -0,0 +1,29 @@ +#pragma once +#include +#include +#include +#include + +namespace eosio { +namespace blockvault { + +class backend { + public: + struct sync_callback { + virtual void on_snapshot(const char* snapshot_filename) = 0; + virtual void on_block(std::string_view block) = 0; + }; + + virtual ~backend() {} + + virtual bool propose_constructed_block(std::pair watermark, uint32_t lib, + const std::vector& block_content, std::string_view block_id, + std::string_view previous_block_id) = 0; + virtual bool append_external_block(uint32_t block_num, uint32_t lib, const std::vector& block_content, + std::string_view block_id, std::string_view previous_block_id) = 0; + virtual bool propose_snapshot(std::pair watermark, const char* snapshot_filename) = 0; + virtual void sync(std::string_view block_id, sync_callback& callback) = 0; +}; + +} // namespace blockvault +} // namespace eosio diff --git a/plugins/blockvault_client_plugin/block_vault_impl.hpp b/plugins/blockvault_client_plugin/block_vault_impl.hpp new file mode 100644 index 00000000000..c74bd20919b --- /dev/null +++ b/plugins/blockvault_client_plugin/block_vault_impl.hpp @@ -0,0 +1,146 @@ +#pragma once + +#include "backend.hpp" +#include +#include +#include +#include +#include +#include + +namespace eosio { +namespace blockvault { + +template +struct transform_callback : backend::sync_callback { + Compressor& compressor; + eosio::blockvault::sync_callback& target; + + transform_callback(Compressor& comp, eosio::blockvault::sync_callback& t) + : compressor(comp) + , target(t) {} + + void on_snapshot(const char* filename) override { + auto uncompressed_file = compressor.decompress(filename); + + if (uncompressed_file.size()) + target.on_snapshot(uncompressed_file.c_str()); + else + FC_THROW_EXCEPTION(chain::snapshot_decompress_exception, "Unable to decompress snapshot received from block vault"); + } + + void on_block(std::string_view block) override { + chain::signed_block_ptr b = std::make_shared(); + fc::datastream ds(block.cbegin(), block.size()); + fc::raw::unpack(ds, *b); + target.on_block(b); + } +}; + +template +class block_vault_impl : public block_vault_interface { + Compressor compressor; + std::unique_ptr backend; + + boost::asio::io_context ioc; + std::thread thr; + boost::asio::executor_work_guard work; + std::atomic syncing; + fc::logger log; + + public: + block_vault_impl(std::unique_ptr&& be) + : backend(std::move(be)) + , work(boost::asio::make_work_guard(ioc)) {} + + ~block_vault_impl() { + if (thr.joinable()) { + thr.join(); + } + } + + void async_propose_constructed_block(uint32_t lib, chain::signed_block_ptr block, + std::function handler) override { + boost::asio::post(ioc, [this, handler, lib, block]() { + try { + fc::datastream> stream; + fc::raw::pack(stream, *block); + eosio::chain::block_id_type block_id = block->calculate_id(); + + bool r = backend->propose_constructed_block({block->block_num(), block->timestamp.slot}, lib, + stream.storage(), {block_id.data(), block_id.data_size()}, + {block->previous.data(), block->previous.data_size()}); + + // Notice : + // This following logging line is used for checking double production in 'tests/blockvault_tests.py'. + // Make sure the corresponding code in 'tests/blockvault_tests.py' is changed if the format is changed. + fc_dlog(log, "propose_constructed_block(watermark={${bn}, ${ts}}, lib=${lib}) returns ${r}", + ("bn", block->block_num())("ts", block->timestamp.slot)("lib", lib)("r", r)); + handler(r); + } catch (std::exception& ex) { + fc_elog(log, ex.what()); + handler(false); + } + }); + } + void async_append_external_block(uint32_t lib, chain::signed_block_ptr block, + std::function handler) override { + if (syncing.load()) + return; + + boost::asio::post(ioc, [this, handler, lib, block]() { + try { + fc::datastream> stream; + fc::raw::pack(stream, *block); + eosio::chain::block_id_type block_id = block->calculate_id(); + + bool r = backend->append_external_block(block->block_num(), lib, stream.storage(), + {block_id.data(), block_id.data_size()}, + {block->previous.data(), block->previous.data_size()}); + fc_dlog(log, "append_external_block(block_num=${bn}, lib=${lib}) returns ${r}", + ("bn", block->block_num())("lib", lib)("r", r)); + handler(r); + } catch (std::exception& ex) { + fc_elog(log, ex.what()); + handler(false); + } + }); + } + + bool propose_snapshot(watermark_t watermark, const char* snapshot_filename) override { + fc_dlog(log, "propose_snapshot(watermark={${wf}, ${ws}), ${fn})", + ("wf", watermark.first)("ws", watermark.second.slot)("fn", snapshot_filename)); + + std::string compressed_file = compressor.compress(snapshot_filename); + if (compressed_file.size()) { + auto on_exit = fc::make_scoped_exit([&compressed_file]() { boost::filesystem::remove(compressed_file); }); + return backend->propose_snapshot({watermark.first, watermark.second.slot}, compressed_file.c_str()); + } else { + fc_elog(log, "snapshot compress failed"); + return false; + } + } + void sync(const eosio::chain::block_id_type* ptr_block_id, sync_callback& callback) override { + transform_callback cb{compressor, callback}; + std::string_view bid = ptr_block_id ? std::string_view{ptr_block_id->data(), ptr_block_id->data_size()} + : std::string_view{nullptr, 0}; + syncing.store(true); + auto on_exit = fc::make_scoped_exit([this]() { syncing.store(false); }); + backend->sync(bid, cb); + } + + void start() { + thr = std::thread([&log = log, & ioc = ioc]() { + fc_ilog(log, "block vault thread started"); + ioc.run(); + }); + } + + void stop() { work.reset(); } + + void set_logger_name(const char* logger_name) { + fc::logger::update( logger_name, log ); + } +}; +} // namespace blockvault +} // namespace eosio \ No newline at end of file diff --git a/plugins/blockvault_client_plugin/blockvault_client_plugin.cpp b/plugins/blockvault_client_plugin/blockvault_client_plugin.cpp new file mode 100644 index 00000000000..11e66011696 --- /dev/null +++ b/plugins/blockvault_client_plugin/blockvault_client_plugin.cpp @@ -0,0 +1,70 @@ +#include "block_vault_impl.hpp" +#include "zlib_compressor.hpp" +#include +#include // eosio::blockvault_client_plugin +#include // FC_LOG_MESSAGE +#include // std::vector +#if HAS_PQXX +#include "postgres_backend.hpp" +#endif + +namespace eosio { + +static appbase::abstract_plugin& _blockvault_client_plugin = app().register_plugin(); + +using vault_impl = eosio::blockvault::block_vault_impl; +class blockvault_client_plugin_impl : public vault_impl { + public: + blockvault_client_plugin_impl(std::unique_ptr&& be) + : vault_impl(std::move(be)) {} +}; + +blockvault_client_plugin::blockvault_client_plugin() {} + +blockvault_client_plugin::~blockvault_client_plugin() {} + +void blockvault_client_plugin::set_program_options(options_description&, options_description& cfg) { +#ifdef HAS_PQXX + cfg.add_options()("block-vault-backend", bpo::value(), + "the uri for block vault backend. Currently, only PostgreSQL is supported, the format is " + "'postgresql://username:password@localhost/company'"); +#endif +} + +void blockvault_client_plugin::plugin_initialize(const variables_map& options) { + ilog("initializing blockvault_client plugin"); +#ifdef HAS_PQXX + try { + if (options.count("block-vault-backend")) { + std::string uri = options["block-vault-backend"].as(); + if (boost::starts_with(uri, "postgresql://") || boost::starts_with(uri, "postgres://")) { + my.reset(new blockvault_client_plugin_impl(std::make_unique(uri))); + ilog("blockvault_client plugin started"); + } else if (uri.size()) { + elog("unknown block-vault-backend option, skipping it"); + } + } + } + FC_RETHROW_EXCEPTIONS ( error, "blockvault_client plugin initialization error, please double check if the specified `--block-vault-backend` value is correct") +#endif +} + +void blockvault_client_plugin::plugin_startup() { + handle_sighup(); // setup logging + if (my.get()) + my->start(); +} + +void blockvault_client_plugin::plugin_shutdown() { + if (my.get()) + my->stop(); +} + +eosio::blockvault::block_vault_interface* blockvault_client_plugin::get() { return my.get(); } + +void blockvault_client_plugin::handle_sighup() { + if (my.get()) + my->set_logger_name("blockvault_client_plugin"); +} + +} // namespace eosio diff --git a/plugins/blockvault_client_plugin/include/eosio/blockvault_client_plugin/blockvault.hpp b/plugins/blockvault_client_plugin/include/eosio/blockvault_client_plugin/blockvault.hpp new file mode 100644 index 00000000000..6cf90da5ecc --- /dev/null +++ b/plugins/blockvault_client_plugin/include/eosio/blockvault_client_plugin/blockvault.hpp @@ -0,0 +1,100 @@ +#pragma once +#include +#include +#include +#include + +namespace eosio { +namespace blockvault { +using watermark_t = std::pair; + +class sync_callback { + public: + virtual void on_snapshot(const char* snapshot_filename) = 0; + virtual void on_block(eosio::chain::signed_block_ptr block) = 0; +}; + +class block_vault_interface { + public: + virtual ~block_vault_interface() {} + + /// + /// \brief The primary method for adding constructed blocks to the Block + /// Vault. If a proposed constructed block is accepted, the Block Vault + /// cluster will guarantee that no future proposed blocks will be accepted + /// which is in conflict with this block due to double production or finality + /// violations. If the Block Vault cannot make that guarantee for any reason, + /// it must reject the proposal. + /// + /// Notice that handler would be called from a thread different from the invoker + /// of this member function. + /// + /// + /// \param lib The LIB implied by accepting this block. + /// + /// \param block A signed constructed block. + /// + /// \param handler The callback function to inform caller whether the block has accepted the Block Vault or not. + + virtual void async_propose_constructed_block(uint32_t lib, + eosio::chain::signed_block_ptr block, + std::function handler) = 0; + + /// + /// \brief The primary method for adding externally discovered blocks to the + /// Block Vault. If an external block is accepted, the Block Vault cluster + /// will guarantee that all future nodeos nodes will know about this block OR + /// about an accepted snapshot state that exceeds this block's block height + /// before proposing a constructed block. If the implied LIB of this block + /// conflicts with the Block Vault state, then it will be rejected. If the + /// block is older than the currently available snapshot, it is still + /// accepted but will not affect the Block Vault cluster state. + /// + /// Notice that handler would be called from a thread different from the invoker + /// of this member function. + /// + /// \param lib The LIB implied by accepting this block. + /// + /// \param block A signed externally discovered block. + /// + /// \param handler The callback function to inform caller whether the block has accepted the Block Vault or not. + /// + virtual void async_append_external_block(uint32_t lib, eosio::chain::signed_block_ptr block, + std::function handler) = 0; + + /// + /// \brief The primary method for a nodeos node to offer snapshot data to + /// Block Vault that facilitates log pruning. If a snapshot's height is + /// greater than the current snapshot AND less than or equal to the current + /// implied LIB height, it will be accepted, and Block Vault will be able to + /// prune its state internally. Otherwise, it should reject the proposed + /// snapshot. + /// + /// \param snapshot_filename the filename of snapshot. + /// + /// \param watermark The producer watermark at the block height of this snapshot. + /// + virtual bool propose_snapshot(watermark_t watermark, const char* snapshot_filename) = 0; + + /// + /// \brief The primary method for bringing a new nodeos node into sync with + /// the Block Vault. This is the primary method for bringing a new nodeos + /// node into sync with the Block Vault. Syncing is semi-session based, a + /// syncing nodeos will establish a session with a single Block Vault node, + /// stored on that node for the duration of the syncing process. This + /// session is used to guarantee that the Block Vault node does not prune any + /// data that will be needed to complete this sync process until it is + /// completed OR the session is timed-out. This session is not replicated + /// amongst other Block Vault nodes. As a result, if the syncing process is + /// interrupted due to inability to communicate with the chosen Block Vault + /// node, it must restart from the beginning with a new node. + /// + /// \param block_id The BlockID of the best known final block on the client + /// + /// \param callback The callback object to receive the snapshot and blocks + /// + virtual void sync(const eosio::chain::block_id_type* block_id, sync_callback& callback) = 0; +}; + +} // namespace blockvault +} // namespace eosio diff --git a/plugins/blockvault_client_plugin/include/eosio/blockvault_client_plugin/blockvault_client_plugin.hpp b/plugins/blockvault_client_plugin/include/eosio/blockvault_client_plugin/blockvault_client_plugin.hpp new file mode 100644 index 00000000000..175fc5f7c53 --- /dev/null +++ b/plugins/blockvault_client_plugin/include/eosio/blockvault_client_plugin/blockvault_client_plugin.hpp @@ -0,0 +1,47 @@ +#pragma once + +#include // std::optional +#include // std::pair +#include // appbase::plugin +#include "blockvault.hpp" + +namespace eosio { + +using namespace appbase; +using namespace eosio; +using namespace eosio::chain; + +using block_num = uint32_t; + +class blockvault_client_plugin_impl; + +/// Documentation for blockvault_client_plugin. +/// +/// blockvault_client_plugin is a proposed clustered component in an EOSIO +/// network architecture which provides a replicated durable storage with strong +/// consistency guarantees for all the input required by a redundant cluster of +/// nodeos nodes to achieve the guarantees: +/// +/// Guarantee against double-production of blocks +/// Guarantee against finality violation +/// Guarantee of liveness (ability to make progress as a blockchain) +class blockvault_client_plugin : public appbase::plugin { +public: + blockvault_client_plugin(); + virtual ~blockvault_client_plugin(); + + APPBASE_PLUGIN_REQUIRES() + virtual void set_program_options(options_description&, options_description& cfg) final; + + void plugin_initialize(const variables_map& options); + void plugin_startup(); + void plugin_shutdown(); + eosio::blockvault::block_vault_interface* get(); + + void handle_sighup() override; + + private: + std::unique_ptr my; +}; + +} diff --git a/plugins/blockvault_client_plugin/postgres_backend.cpp b/plugins/blockvault_client_plugin/postgres_backend.cpp new file mode 100644 index 00000000000..01d211a9bb6 --- /dev/null +++ b/plugins/blockvault_client_plugin/postgres_backend.cpp @@ -0,0 +1,210 @@ +#include "postgres_backend.hpp" +#include +#include +#include +#include +#include + +namespace eosio { +namespace blockvault { + +postgres_backend::postgres_backend(const std::string& options) + : conn(options) { + + try { + pqxx::work w(conn); + w.exec("CREATE TABLE IF NOT EXISTS BlockData (watermark_bn bigint, watermark_ts bigint, lib bigint, block_num bigint, " + "block_id bytea UNIQUE, previous_block_id bytea, block oid, block_size bigint);" + "CREATE TABLE IF NOT EXISTS SnapshotData (watermark_bn bigint, watermark_ts bigint, snapshot oid);"); + w.commit(); + } + catch (const pqxx::integrity_constraint_violation&) { + // this would happen when multiple clients try to create the tables at the same time. The first one client should succeed, just ignore it. + } + + conn.prepare("serialize_transaction", "SET TRANSACTION ISOLATION LEVEL SERIALIZABLE;"); + + conn.prepare( + "insert_constructed_block", + "INSERT INTO BlockData (watermark_bn, watermark_ts, lib, block_num, block_id, previous_block_id, block, block_size) " + "SELECT $1, $2, $3, $1, $4, $5, $6, $7 WHERE NOT " + "EXISTS (SELECT * FROM BlockData WHERE (watermark_bn >= $1) OR (watermark_ts >= $2) OR (lib > $3))"); + + conn.prepare( + "insert_external_block", + "INSERT INTO BlockData (watermark_bn, watermark_ts, lib, block_num, block_id, previous_block_id, block, block_size) SELECT " + "COALESCE((SELECT MAX(watermark_bn) FROM BlockData),0), COALESCE((SELECT MAX(watermark_ts) FROM " + "BlockData),0), $2, $1, $3, $4, $5, $6 WHERE NOT " + "EXISTS (SELECT * FROM BlockData WHERE lib >= $1)"); + + conn.prepare("get_block_insertion_result", "SELECT block from BlockData WHERE block=$1"); + + conn.prepare("insert_snapshot", + "INSERT INTO SnapshotData (watermark_bn, watermark_ts, snapshot) SELECT $1, $2, $3 WHERE NOT EXISTS " + "(SELECT * FROM SnapshotData WHERE watermark_bn >= $1 OR watermark_ts >= $2)"); + + conn.prepare("get_snapshot_insertion_result", "SELECT snapshot from SnapshotData WHERE snapshot=$1"); + + conn.prepare("get_sync_watermark", "SELECT watermark_bn, watermark_ts FROM BlockData WHERE " + "previous_block_id = $1 ORDER BY watermark_bn, watermark_ts LIMIT 1"); + + conn.prepare("get_latest_snapshot", "SELECT snapshot FROM SnapshotData " + "ORDER BY watermark_bn DESC, watermark_ts DESC LIMIT 1"); + + conn.prepare("get_blocks_since_watermark", "SELECT block, block_size FROM BlockData WHERE " + "watermark_bn >= $1 AND watermark_ts >= $2" + "ORDER BY block_num"); + + conn.prepare("get_all_blocks", "SELECT block, block_size FROM BlockData"); + + conn.prepare("delete_outdated_block_lo", + "SELECT lo_unlink(r.block) FROM BlockData r WHERE watermark_bn <= $1 OR watermark_ts <= $2;"); + conn.prepare("delete_outdated_block_data", "DELETE FROM BlockData WHERE watermark_bn <= $1 OR watermark_ts <= $2;"); + conn.prepare("delete_outdated_snapshot_lo", + "SELECT lo_unlink(r.snapshot) FROM SnapshotData r WHERE watermark_bn < $1 OR watermark_ts < $2;"); + conn.prepare("delete_outdated_snapshot_data", + "DELETE FROM SnapshotData WHERE watermark_bn < $1 OR watermark_ts < $2;"); + + conn.prepare("has_block", "SELECT COUNT(*) FROM BlockData WHERE block_id = $1"); +} // namespace blockvault + +bool postgres_backend::propose_constructed_block(std::pair watermark, uint32_t lib, + const std::vector& block_content, std::string_view block_id, + std::string_view previous_block_id) { + try { + pqxx::work w(conn); + w.exec_prepared0("serialize_transaction"); + pqxx::largeobjectaccess obj(w); + + obj.write(nullptr, 0); + pqxx::binarystring block_id_blob(block_id.data(), block_id.size()); + pqxx::binarystring previous_block_id_blob(previous_block_id.data(), previous_block_id.size()); + w.exec_prepared0("insert_constructed_block", watermark.first, watermark.second, lib, block_id_blob, + previous_block_id_blob, obj.id(), block_content.size()); + auto r = w.exec_prepared("get_block_insertion_result", obj.id()); + if (!r.empty()) { + obj.write(block_content.data(), block_content.size()); + w.commit(); + return true; + } + } catch (const pqxx::sql_error&) { + } + + return false; +} + +bool postgres_backend::append_external_block(uint32_t block_num, uint32_t lib, const std::vector& block_content, + std::string_view block_id, std::string_view previous_block_id) { + try { + pqxx::work w(conn); + w.exec_prepared0("serialize_transaction"); + pqxx::largeobjectaccess obj(w); + obj.write(nullptr, 0); + pqxx::binarystring block_id_blob(block_id.data(), block_id.size()); + pqxx::binarystring previous_block_id_blob(previous_block_id.data(), previous_block_id.size()); + w.exec_prepared0("insert_external_block", block_num, lib, block_id_blob, previous_block_id_blob, obj.id(), + block_content.size()); + auto r = w.exec_prepared("get_block_insertion_result", obj.id()); + if (!r.empty()) { + obj.write(block_content.data(), block_content.size()); + w.commit(); + return true; + } + } catch (const pqxx::sql_error&) { + } + return false; +} + +bool postgres_backend::propose_snapshot(std::pair watermark, const char* snapshot_filename) { + + try { + std::filebuf infile; + + infile.open(snapshot_filename, std::ios::in); + + pqxx::work w(conn); + w.exec_prepared0("serialize_transaction"); + pqxx::largeobjectaccess obj(w); + obj.write(nullptr, 0); + + w.exec_prepared0("insert_snapshot", watermark.first, watermark.second, obj.id()); + auto r = w.exec_prepared("get_snapshot_insertion_result", obj.id()); + + if (!r.empty()) { + + const int chunk_size = 4096; + char chunk[chunk_size]; + + auto sz = chunk_size; + while (sz == chunk_size) { + sz = infile.sgetn(chunk, chunk_size); + obj.write(chunk, sz); + }; + + w.exec_prepared("delete_outdated_block_lo", watermark.first, watermark.second); + w.exec_prepared("delete_outdated_block_data", watermark.first, watermark.second); + w.exec_prepared("delete_outdated_snapshot_lo", watermark.first, watermark.second); + w.exec_prepared("delete_outdated_snapshot_data", watermark.first, watermark.second); + } + + w.commit(); + return !r.empty(); + + } catch (const pqxx::transaction_rollback&) { + } + + return false; +} + +void retrieve_blocks(backend::sync_callback& callback, pqxx::work& trx, pqxx::result r) { + std::vector block_data; + + for (const auto& x : r) { + pqxx::oid block_oid = x[0].as(); + uint64_t block_size = x[1].as(); + pqxx::largeobjectaccess obj(trx, block_oid, std::ios::in | std::ios::binary); + block_data.resize(block_size); + obj.read(block_data.data(), block_size); + callback.on_block(std::string_view{block_data.data(), block_data.size()}); + } + + trx.commit(); +} + +void postgres_backend::sync(std::string_view previous_block_id, backend::sync_callback& callback) { + pqxx::work trx(conn); + + if (previous_block_id.size()) { + pqxx::binarystring blob(previous_block_id.data(), previous_block_id.size()); + auto r = trx.exec_prepared("get_sync_watermark", blob); + + if (!r.empty()) { + retrieve_blocks( + callback, trx, + trx.exec_prepared("get_blocks_since_watermark", r[0][0].as(), r[0][1].as())); + return; + } + + auto row = trx.exec_prepared1("has_block", blob); + if (row[0].as() != 0) + // in this case, the client is up-to-date, nothing to sync. + return; + } + + auto r = trx.exec_prepared("get_latest_snapshot"); + + if (!r.empty()) { + pqxx::largeobject snapshot_obj(r[0][0].as()); + + fc::temp_file temp_file; + std::string fname = temp_file.path().string(); + + snapshot_obj.to_file(trx, fname.c_str()); + callback.on_snapshot(fname.c_str()); + } + + retrieve_blocks(callback, trx, trx.exec_prepared("get_all_blocks")); +} + +} // namespace blockvault +} // namespace eosio \ No newline at end of file diff --git a/plugins/blockvault_client_plugin/postgres_backend.hpp b/plugins/blockvault_client_plugin/postgres_backend.hpp new file mode 100644 index 00000000000..126fd59122f --- /dev/null +++ b/plugins/blockvault_client_plugin/postgres_backend.hpp @@ -0,0 +1,24 @@ +#pragma once +#include "backend.hpp" +#include + +namespace eosio { +namespace blockvault { + +class postgres_backend : public backend { + public: + pqxx::connection conn; + + postgres_backend(const std::string& options); + + bool propose_constructed_block(std::pair watermark, uint32_t lib, + const std::vector& block_content, std::string_view block_id, + std::string_view previous_block_id) override; + bool append_external_block(uint32_t block_num, uint32_t lib, const std::vector& block_content, + std::string_view block_id, std::string_view previous_block_id) override; + bool propose_snapshot(std::pair watermark, const char* snapshot_filename) override; + void sync(std::string_view previous_block_id, eosio::blockvault::backend::sync_callback& callback) override; +}; + +} // namespace blockvault +} // namespace eosio \ No newline at end of file diff --git a/plugins/blockvault_client_plugin/tests/CMakeLists.txt b/plugins/blockvault_client_plugin/tests/CMakeLists.txt new file mode 100644 index 00000000000..233df775864 --- /dev/null +++ b/plugins/blockvault_client_plugin/tests/CMakeLists.txt @@ -0,0 +1,9 @@ +add_executable(blockvault_unittests main.cpp postgres_backend_tests.cpp compressor_test.cpp) +target_link_libraries(blockvault_unittests blockvault Boost::unit_test_framework) + +execute_process(COMMAND ${CMAKE_SOURCE_DIR}/scripts/postgres_control.sh status OUTPUT_VARIABLE HAS_POSTGRES_SERVER) + +if (HAS_POSTGRES_SERVER) + configure_file(test.sh ${CMAKE_CURRENT_BINARY_DIR}/test.sh @ONLY) + add_test(NAME blockvault_unittests COMMAND test.sh) +endif() diff --git a/plugins/blockvault_client_plugin/tests/compressor_test.cpp b/plugins/blockvault_client_plugin/tests/compressor_test.cpp new file mode 100644 index 00000000000..cc208ccc67e --- /dev/null +++ b/plugins/blockvault_client_plugin/tests/compressor_test.cpp @@ -0,0 +1,29 @@ +#include "../zlib_compressor.hpp" +#include +#include +#include +#include + +BOOST_AUTO_TEST_CASE(zlib_compressor_test) { + + std::vector content(1024 * sizeof(int)); + std::generate(reinterpret_cast(content.data()), reinterpret_cast(content.data() + content.size()), rand); + + fc::temp_file file; + std::filebuf fbuf; + fbuf.open(file.path().string().c_str(), std::ios::out | std::ios::binary); + fbuf.sputn(content.data(), content.size()); + fbuf.close(); + + eosio::blockvault::zlib_compressor compressor; + std::string compressed_file = compressor.compress(file.path().string().c_str()); + auto x = fc::make_scoped_exit([&compressed_file]() { boost::filesystem::remove(compressed_file); }); + + std::string decompressed_file = compressor.decompress(compressed_file.c_str()); + auto y = fc::make_scoped_exit([&decompressed_file]() { boost::filesystem::remove(decompressed_file); }); + + std::ifstream strm(decompressed_file.c_str(), std::ios_base::in | std::ios_base::binary); + + BOOST_CHECK(std::equal(std::istreambuf_iterator(strm), std::istreambuf_iterator(), content.cbegin(), + content.cend())); +} diff --git a/plugins/blockvault_client_plugin/tests/main.cpp b/plugins/blockvault_client_plugin/tests/main.cpp new file mode 100644 index 00000000000..971d8b2d071 --- /dev/null +++ b/plugins/blockvault_client_plugin/tests/main.cpp @@ -0,0 +1,2 @@ +#define BOOST_TEST_MODULE BlockVault +#include \ No newline at end of file diff --git a/plugins/blockvault_client_plugin/tests/postgres_backend_tests.cpp b/plugins/blockvault_client_plugin/tests/postgres_backend_tests.cpp new file mode 100644 index 00000000000..98829c763c5 --- /dev/null +++ b/plugins/blockvault_client_plugin/tests/postgres_backend_tests.cpp @@ -0,0 +1,255 @@ +#include "../postgres_backend.hpp" +#include +#include +#include + +std::vector mock_snapshot_content = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k'}; + +// create a test block_id where the first 4 bytes is block_num and the 5th byte +// is discriminator, the purpose of discriminator is to differentiate block ids with +// the same block_num +struct mock_block_id { + char buf[32]; + mock_block_id(uint32_t block_num, char discriminator) { + memset(buf, 0, 32); + memcpy(buf, &block_num, sizeof(block_num)); + buf[4] = discriminator; + } + + operator std::string_view() const { return std::string_view(buf, 32); } +}; + +template +struct backend_reset; + +template <> +struct backend_reset { + backend_reset() { + pqxx::connection conn; + pqxx::work w(conn); + w.exec("DROP TABLE IF EXISTS BlockData; DROP TABLE IF EXISTS SnapshotData; SELECT lo_unlink(l.oid) FROM " + "pg_largeobject_metadata l;"); + w.commit(); + } +}; + +template +struct backend_test_fixture : backend_reset { + + Backend backend; + fc::temp_file tmp_file; + + backend_test_fixture() + : backend("") { + + std::filebuf fbuf; + fbuf.open(tmp_file.path().string().c_str(), std::ios::out | std::ios::binary); + fbuf.sputn(mock_snapshot_content.data(), mock_snapshot_content.size()); + } + + bool propose_constructed_block(uint32_t watermark_bn, uint32_t watermark_ts, uint32_t lib, + char block_discriminator = 'a', char previous_block_discriminator = 'b') { + std::vector block(64); + block[0] = block_discriminator; + return backend.propose_constructed_block({watermark_bn, watermark_ts}, lib, block, + mock_block_id(watermark_bn, block_discriminator), + mock_block_id(watermark_bn - 1, previous_block_discriminator)); + } + + bool append_external_block(uint32_t block_num, uint32_t lib, char block_discriminator = 'a', + char previous_block_discriminator = 'b') { + std::vector block(64); + block[0] = block_discriminator; + + return backend.append_external_block(block_num, lib, block, mock_block_id{block_num, block_discriminator}, + mock_block_id{block_num - 1, previous_block_discriminator}); + } + + bool propose_snapshot(uint32_t watermark_bn, uint32_t watermark_ts) { + return backend.propose_snapshot({watermark_bn, watermark_ts}, tmp_file.path().string().c_str()); + } + + void sync(std::optional previous_block_id, eosio::blockvault::backend::sync_callback& callback) { + if (previous_block_id) { + backend.sync(*previous_block_id, callback); + } else + backend.sync({0, 0}, callback); + } +}; + +typedef boost::mpl::list test_types; + +BOOST_AUTO_TEST_CASE_TEMPLATE(test_propose_constructed_block, T, test_types) { + backend_test_fixture fixture; + + // watermark_bn and watermark_ts should be increasing, + // lib should be non-decreasing + + BOOST_REQUIRE(fixture.propose_constructed_block(10, 1, 1, 'a')); + BOOST_REQUIRE(fixture.propose_constructed_block(11, 2, 2, 'b')); + BOOST_REQUIRE(fixture.propose_constructed_block(12, 3, 3, 'c')); + + // watermark_ts is not increasing + BOOST_REQUIRE(!fixture.propose_constructed_block(13, 3, 4, 'd')); + + // watermark_bn is not increasing + BOOST_REQUIRE(!fixture.propose_constructed_block(12, 4, 4, 'd')); + + // lib is not increasing + BOOST_REQUIRE(fixture.propose_constructed_block(13, 4, 3, 'd')); + + // watermark_bn, watermark_ts and lib are all increasing + BOOST_REQUIRE(fixture.propose_constructed_block(14, 5, 4, 'e')); + + // lib is decreasing + BOOST_REQUIRE(!fixture.propose_constructed_block(15, 6, 3, 'f')); + + // watermark_bn is decreasing + BOOST_REQUIRE(!fixture.propose_constructed_block(13, 6, 4, 'f')); + + // watermark_ts is decreasing + BOOST_REQUIRE(!fixture.propose_constructed_block(15, 4, 4, 'f')); +} + +BOOST_AUTO_TEST_CASE_TEMPLATE(test_append_external_block, T, test_types) { + + backend_test_fixture fixture; + + BOOST_REQUIRE(fixture.append_external_block(9, 1)); + + BOOST_REQUIRE(fixture.propose_constructed_block(10, 1, 2)); + BOOST_REQUIRE(fixture.propose_constructed_block(11, 2, 3)); + BOOST_REQUIRE(fixture.propose_constructed_block(12, 3, 4)); // watermark (12, 3), max_lib 4 + + // block.block_num > max_lib + BOOST_REQUIRE(fixture.append_external_block(5, 4)); + + // block.block_num == max_lib + BOOST_REQUIRE(!fixture.append_external_block(4, 4)); + + // block.block_num < max_lib + BOOST_REQUIRE(!fixture.append_external_block(2, 4)); + + // block lib > max_lib + BOOST_REQUIRE(fixture.append_external_block(6, 5)); + + BOOST_REQUIRE(fixture.propose_constructed_block(13, 4, 5)); +} + +BOOST_AUTO_TEST_CASE_TEMPLATE(test_propose_snapshot, T, test_types) { + + backend_test_fixture fixture; + + BOOST_REQUIRE(fixture.propose_snapshot(10, 1)); + + // increase both watermark block_num & timestamp + BOOST_REQUIRE(fixture.propose_snapshot(11, 2)); + + // just increase watermark timestamp + BOOST_REQUIRE(!fixture.propose_snapshot(11, 3)); + + // just increase watermark num + BOOST_REQUIRE(!fixture.propose_snapshot(12, 2)); +} + +struct mock_sync_callback : eosio::blockvault::backend::sync_callback { + + std::vector expected_snapshot_content; + std::vector reverse_expected_block_first_bytes; + std::vector received_snapshot_content; + + void on_snapshot(const char* snapshot_filename) override { + std::ifstream is(snapshot_filename); + std::istream_iterator start(is), end; + std::copy(start, end, std::back_inserter(received_snapshot_content)); + } + + void on_block(std::string_view block) override { + BOOST_REQUIRE(reverse_expected_block_first_bytes.size()); + BOOST_CHECK_EQUAL(block[0], reverse_expected_block_first_bytes.back()); + reverse_expected_block_first_bytes.pop_back(); + } + + ~mock_sync_callback() { + BOOST_CHECK(std::equal(expected_snapshot_content.begin(), expected_snapshot_content.end(), + received_snapshot_content.begin())); + BOOST_CHECK(reverse_expected_block_first_bytes.empty()); + } +}; + +BOOST_AUTO_TEST_CASE_TEMPLATE(test_sync, T, test_types) { + + backend_test_fixture fixture; + + BOOST_REQUIRE(fixture.propose_constructed_block(10, 1, 1, 'a', '\0')); // watermark (10, 1), max_lib 1 + BOOST_REQUIRE(fixture.propose_constructed_block(11, 2, 2, 'b', 'a')); // watermark (11, 2), max_lib 2 + BOOST_REQUIRE(fixture.propose_constructed_block(12, 3, 3, 'c', 'b')); // watermark (12, 3), max_lib 3 + BOOST_REQUIRE(fixture.propose_constructed_block(13, 4, 4, 'd', 'c')); // watermark (13, 4), max_lib 4 + + // append two different blocks with the same block numbers + BOOST_REQUIRE(fixture.append_external_block(13, 4, 'e', 'a')); // watermark (13, 4), max_lib 4 + BOOST_REQUIRE(fixture.append_external_block(13, 4, 'f', 'a')); // watermark (13, 4), max_lib 4 + + BOOST_REQUIRE(fixture.propose_constructed_block(14, 5, 5, 'g', 'd')); // watermark (14, 5), max_lib 5 + + // block relationship constructed by above code + // + // (10,1) (11,2) (12,3) (13,4) (14,5) + // + // +-- b ---- c ---- d ---- g + // a --+---------------- e + // +---------------- f + // + + { + mock_sync_callback callback; + callback.reverse_expected_block_first_bytes = {'g', 'f', 'e', 'd', 'c', 'b', 'a'}; + BOOST_REQUIRE_NO_THROW(fixture.sync(std::optional{}, callback)); + } + + { + mock_sync_callback callback; + callback.reverse_expected_block_first_bytes = {'g', 'f', 'e', 'd'}; + BOOST_REQUIRE_NO_THROW(fixture.sync(mock_block_id(12, 'c'), callback)); + } + + { + mock_sync_callback callback; + callback.reverse_expected_block_first_bytes = {'g', 'f', 'e', 'd', 'c', 'b'}; + + // given the previous_block_id is {4, 'a'} , there are 3 blocks have the same previous_block_id + // (i.e. block 'b' and 'e', and 'f'), we should sync every block from the lower waterwark, (i.e. block 'b' ) + BOOST_REQUIRE_NO_THROW(fixture.sync(mock_block_id(10, 'a'), callback)); + } + + BOOST_REQUIRE(fixture.propose_snapshot(12, 1)); + + { + mock_sync_callback callback; + callback.expected_snapshot_content = mock_snapshot_content; + callback.reverse_expected_block_first_bytes = {'g', 'f', 'e', 'd'}; + // sync from empty block id + BOOST_REQUIRE_NO_THROW(fixture.sync(std::optional{}, callback)); + } + + { + mock_sync_callback callback; + callback.expected_snapshot_content = mock_snapshot_content; + callback.reverse_expected_block_first_bytes = {'g', 'f', 'e', 'd'}; + // sync from non-existant block id + BOOST_REQUIRE_NO_THROW(fixture.sync(mock_block_id{12, 't'}, callback)); + } + + { + mock_sync_callback callback; + callback.reverse_expected_block_first_bytes = {'g'}; + // sync from existant block id + BOOST_REQUIRE_NO_THROW(fixture.sync(mock_block_id{13, 'd'}, callback)); + } + + { + mock_sync_callback callback; + // sync from the most recent block id + BOOST_REQUIRE_NO_THROW(fixture.sync(mock_block_id{14, 'g'}, callback)); + } +} diff --git a/plugins/blockvault_client_plugin/tests/test.sh b/plugins/blockvault_client_plugin/tests/test.sh new file mode 100755 index 00000000000..77e0e176832 --- /dev/null +++ b/plugins/blockvault_client_plugin/tests/test.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +postgres_control=@CMAKE_BINARY_DIR@/scripts/postgres_control.sh +$postgres_control start +finish() { + $postgres_control stop +} + +trap finish EXIT + +export PGPORT=${PGPORT-5432} +export PGPASSWORD=password +export PGUSER=postgres +export PGHOST=localhost +@CMAKE_CURRENT_BINARY_DIR@/blockvault_unittests + diff --git a/plugins/blockvault_client_plugin/zlib_compressor.hpp b/plugins/blockvault_client_plugin/zlib_compressor.hpp new file mode 100644 index 00000000000..79857e12fbd --- /dev/null +++ b/plugins/blockvault_client_plugin/zlib_compressor.hpp @@ -0,0 +1,38 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace eosio { +namespace blockvault { + +struct zlib_compressor { + + template + std::string convert(const char* input_filename, const char* output_file_suffix) { + using namespace std; + + std::ifstream infile(input_filename, ios_base::in | ios_base::binary); + boost::iostreams::filtering_streambuf in; + in.push(Filter()); + in.push(infile); + + std::string out_filename = std::string(input_filename) + output_file_suffix; + ofstream outfile(out_filename, ios_base::out | ios_base::binary); + boost::iostreams::copy(in, outfile); + return out_filename; + } + + std::string compress(const char* snapshot_filename) { + return convert(snapshot_filename, ".z"); + } + + std::string decompress(const char* compressed_file) { + return convert(compressed_file, ".bin"); + } +}; + +} // namespace blockvault +} // namespace eosio \ No newline at end of file diff --git a/plugins/chain_api_plugin/chain.swagger.yaml b/plugins/chain_api_plugin/chain.swagger.yaml index 88c055332ac..aeced1228bb 100644 --- a/plugins/chain_api_plugin/chain.swagger.yaml +++ b/plugins/chain_api_plugin/chain.swagger.yaml @@ -1,7 +1,7 @@ openapi: 3.0.0 info: title: Chain API - description: "OAS 3.0 Nodeos [chain_api_plugin](https://eosio.github.io/eos/latest/nodeos/plugins/chain_api_plugin/index) API Specification\r" + description: "OAS 3.0 Nodeos Chain API Specification\r" version: 1.0.0 license: name: MIT @@ -37,14 +37,14 @@ paths: - account_name properties: account_name: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" responses: "200": description: OK content: application/json: schema: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Account.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Account.yaml" /get_block: post: description: Returns an object containing various details about a specific block on the blockchain. @@ -66,7 +66,29 @@ paths: content: application/json: schema: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Block.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Block.yaml" + /get_block_info: + post: + description: Similar to `get_block` but returns a fixed-size smaller subset of the block data. + operationId: get_block_info + requestBody: + content: + application/json: + schema: + type: object + required: + - block_num + properties: + block_num: + type: integer + description: Provide a `block number` + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "https://eosio.github.io/schemata/v2.1/oas/BlockInfo.yaml" /get_info: post: description: Returns an object containing various details about the blockchain. @@ -78,7 +100,7 @@ paths: content: application/json: schema: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Info.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Info.yaml" /push_transaction: post: @@ -94,7 +116,7 @@ paths: type: array description: array of signatures required to authorize transaction items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Signature.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Signature.yaml" compression: type: boolean description: Compression used, usually false @@ -126,7 +148,7 @@ paths: type: array description: array of signatures required to authorize transaction items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Signature.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Signature.yaml" compression: type: boolean description: Compression used, usually false @@ -155,7 +177,7 @@ paths: schema: type: array items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Transaction.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Transaction.yaml" responses: "200": description: OK @@ -186,7 +208,7 @@ paths: content: application/json: schema: - $ref: "https://eosio.github.io/schemata/v2.0/oas/BlockHeaderState.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/BlockHeaderState.yaml" /get_abi: post: @@ -201,14 +223,14 @@ paths: - account_name properties: account_name: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" responses: "200": description: OK content: application/json: schema: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Abi.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Abi.yaml" /get_currency_balance: post: description: Retrieves the current balance @@ -224,11 +246,11 @@ paths: - symbol properties: code: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" account: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" symbol: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Symbol.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Symbol.yaml" responses: "200": @@ -238,7 +260,7 @@ paths: schema: type: array items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Symbol.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Symbol.yaml" /get_currency_stats: post: @@ -251,9 +273,9 @@ paths: type: object properties: code: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" symbol: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Symbol.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Symbol.yaml" responses: "200": @@ -277,12 +299,12 @@ paths: - available_keys properties: transaction: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Transaction.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Transaction.yaml" available_keys: type: array description: Provide the available keys items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/PublicKey.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/PublicKey.yaml" responses: "200": description: OK @@ -334,17 +356,17 @@ paths: type: array nullable: true items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/ProducerSchedule.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/ProducerSchedule.yaml" pending: type: array nullable: true items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/ProducerSchedule.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/ProducerSchedule.yaml" proposed: type: array nullable: true items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/ProducerSchedule.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/ProducerSchedule.yaml" /get_raw_code_and_abi: @@ -360,7 +382,7 @@ paths: - account_name properties: account_name: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" responses: "200": @@ -371,7 +393,7 @@ paths: type: object properties: account_name: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" wasm: type: string description: base64 encoded wasm @@ -390,7 +412,7 @@ paths: type: object properties: lower_bound: - $ref: "https://eosio.github.io/schemata/v2.0/oas/DateTimeSeconds.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/DateTimeSeconds.yaml" limit: description: The maximum number of transactions to return type: integer @@ -408,7 +430,7 @@ paths: transactions: type: array items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Transaction.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Transaction.yaml" /get_table_by_scope: @@ -459,9 +481,9 @@ paths: rows: type: array items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/TableScope.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/TableScope.yaml" more: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" /get_table_rows: post: @@ -526,6 +548,60 @@ paths: type: array items: {} + /get_kv_table_rows: + post: + description: Returns an object containing rows from the specified kv table. + operationId: get_kv_table_rows + requestBody: + content: + application/json: + schema: + type: object + required: + - code + - table + - index_name + properties: + code: + type: string + description: The name of the smart contract that controls the provided kv table + table: + type: string + description: The name of the kv table to query + index_name: + type: string + description: The name of primary or secondary index + encode_type: + type: string ( 'bytes' for arbitray binary index value ) + description: Type of key specified by index_position (for example - `uint64_t` or `name`) + index_value: + type: string + description: index value used for point query encoded as 'encode_type' + lower_bound: + type: string + description: lower bound index value for ranged query. Query result includes rows specified with lower_bound + upper_bound: + type: string + description: upper bound index value for ranged query. Query result doesn't include rows specified with upper_bound + limit: + type: integer + description: Limit number of results returned. + format: int32 + reverse: + type: boolean + description: Reverse the order of returned results + responses: + "200": + description: OK + content: + application/json: + schema: + type: object + properties: + rows: + type: array + items: {} + /abi_json_to_bin: post: description: Returns an object containing the serialized action data. @@ -541,9 +617,9 @@ paths: - args properties: code: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" action: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" args: type: object description: json object of the action parameters that will be serialized. @@ -557,7 +633,7 @@ paths: type: object properties: binargs: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Hex.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Hex.yaml" /abi_bin_to_json: post: @@ -575,11 +651,11 @@ paths: - binargs properties: code: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" action: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" binargs: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Hex.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Hex.yaml" responses: "200": description: OK @@ -602,7 +678,7 @@ paths: - code_as_wasm properties: account_name: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" code_as_wasm: type: integer default: 1 @@ -617,15 +693,15 @@ paths: title: GetCodeResponse.yaml properties: name: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" code_hash: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Sha256.yaml" wast: type: string wasm: type: string abi: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Abi.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Abi.yaml" /get_raw_abi: post: @@ -640,7 +716,7 @@ paths: - account_name properties: account_name: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" responses: "200": description: OK @@ -650,12 +726,12 @@ paths: type: object properties: account_name: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" code_hash: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Sha256.yaml" abi_hash: allOf: - - $ref: "https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml" + - $ref: "https://eosio.github.io/schemata/v2.1/oas/Sha256.yaml" abi: type: string @@ -728,13 +804,13 @@ paths: description: List of authorizing accounts and/or actor/permissions items: anyOf: - - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" - - $ref: "https://eosio.github.io/schemata/v2.0/oas/Authority.yaml" + - $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" + - $ref: "https://eosio.github.io/schemata/v2.1/oas/Authority.yaml" keys: type: array description: List of authorizing keys items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/PublicKey.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/PublicKey.yaml" responses: "200": description: OK @@ -760,13 +836,13 @@ paths: - threshold properties: account_name: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" permission_name: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" authorizer: oneOf: - - $ref: "https://eosio.github.io/schemata/v2.0/oas/PublicKey.yaml" - - $ref: "https://eosio.github.io/schemata/v2.0/oas/Authority.yaml" + - $ref: "https://eosio.github.io/schemata/v2.1/oas/PublicKey.yaml" + - $ref: "https://eosio.github.io/schemata/v2.1/oas/Authority.yaml" weight: type: "integer" description: the weight that this authorizer adds to satisfy the permission diff --git a/plugins/chain_api_plugin/chain_api_plugin.cpp b/plugins/chain_api_plugin/chain_api_plugin.cpp index 44d5196f575..05296629af1 100644 --- a/plugins/chain_api_plugin/chain_api_plugin.cpp +++ b/plugins/chain_api_plugin/chain_api_plugin.cpp @@ -48,25 +48,12 @@ namespace { } } -#define CALL(api_name, api_handle, api_namespace, call_name, http_response_code) \ +#define CALL_WITH_400(api_name, api_handle, api_namespace, call_name, http_response_code, params_type) \ {std::string("/v1/" #api_name "/" #call_name), \ [api_handle](string, string body, url_response_callback cb) mutable { \ api_handle.validate(); \ try { \ - if (body.empty()) body = "{}"; \ - fc::variant result( api_handle.call_name(fc::json::from_string(body).as()) ); \ - cb(http_response_code, std::move(result)); \ - } catch (...) { \ - http_plugin::handle_exception(#api_name, #call_name, body, cb); \ - } \ - }} - -#define CALL_WITH_400(api_name, api_handle, api_namespace, call_name, http_response_code) \ -{std::string("/v1/" #api_name "/" #call_name), \ - [api_handle](string, string body, url_response_callback cb) mutable { \ - api_handle.validate(); \ - try { \ - auto params = parse_params(body);\ + auto params = parse_params(body);\ fc::variant result( api_handle.call_name( std::move(params) ) ); \ cb(http_response_code, std::move(result)); \ } catch (...) { \ @@ -74,75 +61,83 @@ namespace { } \ }} -#define CALL_ASYNC(api_name, api_handle, api_namespace, call_name, call_result, http_response_code) \ +#define CALL_ASYNC_WITH_400(api_name, api_handle, api_namespace, call_name, call_result, http_response_code, params_type) \ {std::string("/v1/" #api_name "/" #call_name), \ [api_handle](string, string body, url_response_callback cb) mutable { \ - if (body.empty()) body = "{}"; \ api_handle.validate(); \ - api_handle.call_name(fc::json::from_string(body).as(),\ - [cb, body](const fc::static_variant& result){\ - if (result.contains()) {\ - try {\ - result.get()->dynamic_rethrow_exception();\ - } catch (...) {\ - http_plugin::handle_exception(#api_name, #call_name, body, cb);\ + try { \ + auto params = parse_params(body);\ + api_handle.call_name( std::move(params),\ + [cb, body](const std::variant& result){\ + if (std::holds_alternative(result)) {\ + try {\ + std::get(result)->dynamic_rethrow_exception();\ + } catch (...) {\ + http_plugin::handle_exception(#api_name, #call_name, body, cb);\ + }\ + } else {\ + cb(http_response_code, std::visit(async_result_visitor(), result));\ }\ - } else {\ - cb(http_response_code, result.visit(async_result_visitor()));\ - }\ - });\ + });\ + } catch (...) { \ + http_plugin::handle_exception(#api_name, #call_name, body, cb); \ + } \ }\ } -#define CHAIN_RO_CALL(call_name, http_response_code) CALL(chain, ro_api, chain_apis::read_only, call_name, http_response_code) -#define CHAIN_RW_CALL(call_name, http_response_code) CALL(chain, rw_api, chain_apis::read_write, call_name, http_response_code) -#define CHAIN_RO_CALL_ASYNC(call_name, call_result, http_response_code) CALL_ASYNC(chain, ro_api, chain_apis::read_only, call_name, call_result, http_response_code) -#define CHAIN_RW_CALL_ASYNC(call_name, call_result, http_response_code) CALL_ASYNC(chain, rw_api, chain_apis::read_write, call_name, call_result, http_response_code) +#define CHAIN_RO_CALL(call_name, http_response_code, params_type) CALL_WITH_400(chain, ro_api, chain_apis::read_only, call_name, http_response_code, params_type) +#define CHAIN_RW_CALL(call_name, http_response_code, params_type) CALL_WITH_400(chain, rw_api, chain_apis::read_write, call_name, http_response_code, params_type) +#define CHAIN_RO_CALL_ASYNC(call_name, call_result, http_response_code, params_type) CALL_ASYNC_WITH_400(chain, ro_api, chain_apis::read_only, call_name, call_result, http_response_code, params_type) +#define CHAIN_RW_CALL_ASYNC(call_name, call_result, http_response_code, params_type) CALL_ASYNC_WITH_400(chain, rw_api, chain_apis::read_write, call_name, call_result, http_response_code, params_type) -#define CHAIN_RO_CALL_WITH_400(call_name, http_response_code) CALL_WITH_400(chain, ro_api, chain_apis::read_only, call_name, http_response_code) +#define CHAIN_RO_CALL_WITH_400(call_name, http_response_code, params_type) CALL_WITH_400(chain, ro_api, chain_apis::read_only, call_name, http_response_code, params_type) + + void chain_api_plugin::plugin_startup() { ilog( "starting chain_api_plugin" ); my.reset(new chain_api_plugin_impl(app().get_plugin().chain())); auto& chain = app().get_plugin(); auto ro_api = chain.get_read_only_api(); auto rw_api = chain.get_read_write_api(); - + auto& _http_plugin = app().get_plugin(); ro_api.set_shorten_abi_errors( !_http_plugin.verbose_errors() ); _http_plugin.add_api({ - CHAIN_RO_CALL(get_info, 200)}, appbase::priority::medium_high); + CHAIN_RO_CALL(get_info, 200, http_params_types::no_params_required)}, appbase::priority::medium_high); _http_plugin.add_api({ - CHAIN_RO_CALL(get_activated_protocol_features, 200), - CHAIN_RO_CALL(get_block, 200), - CHAIN_RO_CALL(get_block_header_state, 200), - CHAIN_RO_CALL(get_account, 200), - CHAIN_RO_CALL(get_code, 200), - CHAIN_RO_CALL(get_code_hash, 200), - CHAIN_RO_CALL(get_abi, 200), - CHAIN_RO_CALL(get_raw_code_and_abi, 200), - CHAIN_RO_CALL(get_raw_abi, 200), - CHAIN_RO_CALL(get_table_rows, 200), - CHAIN_RO_CALL(get_table_by_scope, 200), - CHAIN_RO_CALL(get_currency_balance, 200), - CHAIN_RO_CALL(get_currency_stats, 200), - CHAIN_RO_CALL(get_producers, 200), - CHAIN_RO_CALL(get_producer_schedule, 200), - CHAIN_RO_CALL(get_scheduled_transactions, 200), - CHAIN_RO_CALL(abi_json_to_bin, 200), - CHAIN_RO_CALL(abi_bin_to_json, 200), - CHAIN_RO_CALL(get_required_keys, 200), - CHAIN_RO_CALL(get_transaction_id, 200), - CHAIN_RW_CALL_ASYNC(push_block, chain_apis::read_write::push_block_results, 202), - CHAIN_RW_CALL_ASYNC(push_transaction, chain_apis::read_write::push_transaction_results, 202), - CHAIN_RW_CALL_ASYNC(push_transactions, chain_apis::read_write::push_transactions_results, 202), - CHAIN_RW_CALL_ASYNC(send_transaction, chain_apis::read_write::send_transaction_results, 202) + CHAIN_RO_CALL(get_activated_protocol_features, 200, http_params_types::possible_no_params), + CHAIN_RO_CALL(get_block, 200, http_params_types::params_required), + CHAIN_RO_CALL(get_block_info, 200, http_params_types::params_required), + CHAIN_RO_CALL(get_block_header_state, 200, http_params_types::params_required), + CHAIN_RO_CALL(get_account, 200, http_params_types::params_required), + CHAIN_RO_CALL(get_code, 200, http_params_types::params_required), + CHAIN_RO_CALL(get_code_hash, 200, http_params_types::params_required), + CHAIN_RO_CALL(get_abi, 200, http_params_types::params_required), + CHAIN_RO_CALL(get_raw_code_and_abi, 200, http_params_types::params_required), + CHAIN_RO_CALL(get_raw_abi, 200, http_params_types::params_required), + CHAIN_RO_CALL(get_table_rows, 200, http_params_types::params_required), + CHAIN_RO_CALL(get_kv_table_rows, 200, http_params_types::params_required), + CHAIN_RO_CALL(get_table_by_scope, 200, http_params_types::params_required), + CHAIN_RO_CALL(get_currency_balance, 200, http_params_types::params_required), + CHAIN_RO_CALL(get_currency_stats, 200, http_params_types::params_required), + CHAIN_RO_CALL(get_producers, 200, http_params_types::params_required), + CHAIN_RO_CALL(get_producer_schedule, 200, http_params_types::no_params_required), + CHAIN_RO_CALL(get_scheduled_transactions, 200, http_params_types::params_required), + CHAIN_RO_CALL(abi_json_to_bin, 200, http_params_types::params_required), + CHAIN_RO_CALL(abi_bin_to_json, 200, http_params_types::params_required), + CHAIN_RO_CALL(get_required_keys, 200, http_params_types::params_required), + CHAIN_RO_CALL(get_transaction_id, 200, http_params_types::params_required), + CHAIN_RW_CALL_ASYNC(push_block, chain_apis::read_write::push_block_results, 202, http_params_types::params_required), + CHAIN_RW_CALL_ASYNC(push_transaction, chain_apis::read_write::push_transaction_results, 202, http_params_types::params_required), + CHAIN_RW_CALL_ASYNC(push_transactions, chain_apis::read_write::push_transactions_results, 202, http_params_types::params_required), + CHAIN_RW_CALL_ASYNC(send_transaction, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required) }); - + if (chain.account_queries_enabled()) { _http_plugin.add_async_api({ - CHAIN_RO_CALL_WITH_400(get_accounts_by_authorizers, 200), + CHAIN_RO_CALL_WITH_400(get_accounts_by_authorizers, 200, http_params_types::params_required), }); } } diff --git a/plugins/chain_interface/include/eosio/chain/plugin_interface.hpp b/plugins/chain_interface/include/eosio/chain/plugin_interface.hpp index 9c1186d4eb7..bd5d950172b 100644 --- a/plugins/chain_interface/include/eosio/chain/plugin_interface.hpp +++ b/plugins/chain_interface/include/eosio/chain/plugin_interface.hpp @@ -13,7 +13,7 @@ namespace eosio { namespace chain { namespace plugin_interface { using namespace appbase; template - using next_function = std::function&)>; + using next_function = std::function&)>; struct chain_plugin_interface; @@ -45,6 +45,7 @@ namespace eosio { namespace chain { namespace plugin_interface { namespace methods { // synchronously push a block/trx to a single provider using block_sync = method_decl&), first_provider_policy>; + using blockvault_sync = method_decl; using transaction_async = method_decl), first_provider_policy>; } } diff --git a/plugins/chain_plugin/CMakeLists.txt b/plugins/chain_plugin/CMakeLists.txt index 85603ef92e2..17e2dde294f 100644 --- a/plugins/chain_plugin/CMakeLists.txt +++ b/plugins/chain_plugin/CMakeLists.txt @@ -4,7 +4,12 @@ add_library( chain_plugin chain_plugin.cpp ${HEADERS} ) -target_link_libraries( chain_plugin producer_plugin eosio_chain appbase ) -target_include_directories( chain_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/../chain_interface/include" "${CMAKE_CURRENT_SOURCE_DIR}/../../libraries/appbase/include" "${CMAKE_CURRENT_SOURCE_DIR}/../producer_plugin/include") +if(EOSIO_ENABLE_DEVELOPER_OPTIONS) + message(WARNING "EOSIO Developer Options are enabled; these are NOT supported") + target_compile_definitions(chain_plugin PUBLIC EOSIO_DEVELOPER) +endif() + +target_link_libraries( chain_plugin eosio_chain appbase resource_monitor_plugin blockvault_client_plugin ) +target_include_directories( chain_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/../chain_interface/include" "${CMAKE_CURRENT_SOURCE_DIR}/../../libraries/appbase/include" "${CMAKE_CURRENT_SOURCE_DIR}/../resource_monitor_plugin/include" "${CMAKE_CURRENT_SOURCE_DIR}/../../libraries/abieos/include") add_subdirectory( test ) diff --git a/plugins/chain_plugin/account_query_db.cpp b/plugins/chain_plugin/account_query_db.cpp index 7191c571d34..1f74c1d3610 100644 --- a/plugins/chain_plugin/account_query_db.cpp +++ b/plugins/chain_plugin/account_query_db.cpp @@ -16,6 +16,7 @@ #include using namespace eosio; +using namespace eosio::chain::literals; using namespace boost::multi_index; using namespace boost::bimaps; @@ -68,7 +69,7 @@ namespace { if (p->action_traces.empty()) return false; const auto& act = p->action_traces[0].act; - if (act.account != eosio::chain::config::system_account_name || act.name != N(onblock) || + if (act.account != eosio::chain::config::system_account_name || act.name != "onblock"_n || act.authorization.size() != 1) return false; const auto& auth = act.authorization[0]; @@ -91,7 +92,7 @@ namespace { }; template - auto make_optional_authorizer(const Input& authorizer) -> fc::optional { + auto make_optional_authorizer(const Input& authorizer) -> std::optional { if constexpr (std::is_same_v) { return authorizer; } else { @@ -309,7 +310,7 @@ namespace eosio::chain_apis { permission_set_t deleted; /** - * process traces to find `updateauth` and `deleteauth` calls maintaining a final set of + * process traces to find `updateauth`, `deleteauth` and `newaccount` calls maintaining a final set of * permissions to either update or delete. Intra-block changes are discarded */ auto process_trace = [&](const chain::transaction_trace_ptr& trace) { @@ -327,9 +328,9 @@ namespace eosio::chain_apis { auto itr = deleted.emplace(chain::permission_level{data.account, data.permission}).first; updated.erase(*itr); } else if (at.act.name == chain::newaccount::get_name()) { - auto data = at.act.data_as(); - updated.emplace(chain::permission_level{data.name, N(owner)}); - updated.emplace(chain::permission_level{data.name, N(active)}); + auto data = at.act.data_as(); + updated.emplace(chain::permission_level{data.name, "owner"_n}); + updated.emplace(chain::permission_level{data.name, "active"_n}); } } }; @@ -339,10 +340,10 @@ namespace eosio::chain_apis { for( const auto& r : bsp->block->transactions ) { chain::transaction_id_type id; - if( r.trx.contains()) { - id = r.trx.get(); + if( std::holds_alternative( r.trx ) ) { + id = std::get( r.trx ); } else { - id = r.trx.get().id(); + id = std::get( r.trx ).id(); } const auto it = cached_trace_map.find( id ); diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index a5cd15e9cee..e6994a3918b 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include @@ -13,17 +13,24 @@ #include #include #include +#include +#include +#include #include +#include +#include #include #include #include +#include #include #include #include +#include #include #include @@ -34,6 +41,9 @@ FC_REFLECT_ENUM( chainbase::environment::arch_t, (ARCH_X86_64)(ARCH_ARM)(ARCH_RISCV)(ARCH_OTHER) ) FC_REFLECT(chainbase::environment, (debug)(os)(arch)(boost_version)(compiler) ) +const fc::string deep_mind_logger_name("deep-mind"); +fc::logger _deep_mind_log; + namespace eosio { //declare operator<< and validate funciton for read_mode in the same namespace as read_mode itself @@ -113,6 +123,38 @@ void validate(boost::any& v, } } +std::ostream& operator<<(std::ostream& osm, eosio::chain::backing_store_type b) { + if ( b == eosio::chain::backing_store_type::CHAINBASE ) { + osm << "chainbase"; + } else if ( b == eosio::chain::backing_store_type::ROCKSDB ) { + osm << "rocksdb"; + } + + return osm; +} + +void validate(boost::any& v, + const std::vector& values, + eosio::chain::backing_store_type* /* target_type */, + int) +{ + using namespace boost::program_options; + + // Make sure no previous assignment to 'v' was made. + validators::check_first_occurrence(v); + + // Extract the first string from 'values'. If there is more than + // one string, it's an error, and exception will be thrown. + std::string const& s = validators::get_single_string(values); + + if ( s == "chainbase" ) { + v = boost::any(eosio::chain::backing_store_type::CHAINBASE); + } else if ( s == "rocksdb" ) { + v = boost::any(eosio::chain::backing_store_type::ROCKSDB); + } else { + throw validation_error(validation_error::invalid_option_value); + } +} } using namespace eosio; @@ -135,6 +177,7 @@ class chain_plugin_impl { ,applied_transaction_channel(app().get_channel()) ,incoming_block_channel(app().get_channel()) ,incoming_block_sync_method(app().get_method()) + ,incoming_blockvault_sync_method(app().get_method()) ,incoming_transaction_async_method(app().get_method()) {} @@ -145,16 +188,14 @@ class chain_plugin_impl { bool api_accept_transactions = true; bool account_queries_enabled = false; - - fc::optional fork_db; - fc::optional block_logger; - fc::optional chain_config; - fc::optional chain; - fc::optional genesis; + std::optional fork_db; + std::optional chain_config; + std::optional chain; + std::optional genesis; //txn_msg_rate_limits rate_limits; - fc::optional wasm_runtime; - fc::microseconds abi_serializer_max_time_us; - fc::optional snapshot_path; + std::optional wasm_runtime; + fc::microseconds abi_serializer_max_time_us; + std::optional snapshot_path; // retained references to channels for easy publication @@ -164,10 +205,11 @@ class chain_plugin_impl { channels::irreversible_block::channel_type& irreversible_block_channel; channels::accepted_transaction::channel_type& accepted_transaction_channel; channels::applied_transaction::channel_type& applied_transaction_channel; - incoming::channels::block::channel_type& incoming_block_channel; + incoming::channels::block::channel_type& incoming_block_channel; // retained references to methods for easy calling incoming::methods::block_sync::method_type& incoming_block_sync_method; + incoming::methods::blockvault_sync::method_type& incoming_blockvault_sync_method; incoming::methods::transaction_async::method_type& incoming_transaction_async_method; // method provider handles @@ -177,22 +219,29 @@ class chain_plugin_impl { methods::get_last_irreversible_block_number::method_type::handle get_last_irreversible_block_number_provider; // scoped connections for chain controller - fc::optional pre_accepted_block_connection; - fc::optional accepted_block_header_connection; - fc::optional accepted_block_connection; - fc::optional irreversible_block_connection; - fc::optional accepted_transaction_connection; - fc::optional applied_transaction_connection; - - - fc::optional _account_query_db; - const producer_plugin* producer_plug; + std::optional pre_accepted_block_connection; + std::optional accepted_block_header_connection; + std::optional accepted_block_connection; + std::optional irreversible_block_connection; + std::optional accepted_transaction_connection; + std::optional applied_transaction_connection; + + std::optional _account_query_db; + + void do_non_snapshot_startup(std::function shutdown, std::function check_shutdown) { + if (genesis) { + chain->startup(shutdown, check_shutdown, *genesis); + }else { + chain->startup(shutdown, check_shutdown); + } + } }; chain_plugin::chain_plugin() -:my(new chain_plugin_impl()) { + : my(new chain_plugin_impl()) { app().register_config_type(); app().register_config_type(); + app().register_config_type(); app().register_config_type(); app().register_config_type(); } @@ -205,19 +254,16 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip std::string wasm_runtime_opt = "Override default WASM runtime ("; std::string wasm_runtime_desc; std::string delim; - - wasm_runtime_opt += " \"wabt\""; - wasm_runtime_desc += "\"wabt\" : The WebAssembly Binary Toolkit.\n"; - delim = ", "; - #ifdef EOSIO_EOS_VM_JIT_RUNTIME_ENABLED - wasm_runtime_opt += delim + " \"eos-vm-jit\""; + wasm_runtime_opt += " \"eos-vm-jit\""; wasm_runtime_desc += "\"eos-vm-jit\" : A WebAssembly runtime that compiles WebAssembly code to native x86 code prior to execution.\n"; + delim = ", "; #endif #ifdef EOSIO_EOS_VM_RUNTIME_ENABLED wasm_runtime_opt += delim + "\"eos-vm\""; wasm_runtime_desc += "\"eos-vm\" : A WebAssembly interpreter.\n"; + delim = ", "; #endif #ifdef EOSIO_EOS_VM_OC_DEVELOPER @@ -231,6 +277,25 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip cfg.add_options() ("blocks-dir", bpo::value()->default_value("blocks"), "the location of the blocks directory (absolute path or relative to application data dir)") + ("blocks-log-stride", bpo::value()->default_value(config::default_blocks_log_stride), + "split the block log file when the head block number is the multiple of the stride\n" + "When the stride is reached, the current block log and index will be renamed '/blocks--.log/index'\n" + "and a new current block log and index will be created with the most recent block. All files following\n" + "this format will be used to construct an extended block log.") + ("max-retained-block-files", bpo::value()->default_value(config::default_max_retained_block_files), + "the maximum number of blocks files to retain so that the blocks in those files can be queried.\n" + "When the number is reached, the oldest block file would be moved to archive dir or deleted if the archive dir is empty.\n" + "The retained block log files should not be manipulated by users." ) + ("blocks-retained-dir", bpo::value()->default_value(""), + "the location of the blocks retained directory (absolute path or relative to blocks dir).\n" + "If the value is empty, it is set to the value of blocks dir.") + ("blocks-archive-dir", bpo::value()->default_value(config::default_blocks_archive_dir_name), + "the location of the blocks archive directory (absolute path or relative to blocks dir).\n" + "If the value is empty, blocks files beyond the retained limit will be deleted.\n" + "All files in the archive directory are completely under user's control, i.e. they won't be accessed by nodeos anymore.") + ("fix-irreversible-blocks", bpo::value()->default_value("false"), + "When the existing block log is inconsistent with the index, allows fixing the block log and index files automatically - that is, " + "it will take the highest indexed block if it is valid; otherwise it will repair the block log and reconstruct the index.") ("protocol-features-dir", bpo::value()->default_value("protocol_features"), "the location of the protocol_features directory (absolute path or relative to application config dir)") ("checkpoint", bpo::value>()->composing(), "Pairs of [BLOCK_NUM,BLOCK_ID] that should be enforced as checkpoints.") @@ -248,6 +313,19 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip "Override default maximum ABI serialization time allowed in ms") ("chain-state-db-size-mb", bpo::value()->default_value(config::default_state_size / (1024 * 1024)), "Maximum size (in MiB) of the chain state database") ("chain-state-db-guard-size-mb", bpo::value()->default_value(config::default_state_guard_size / (1024 * 1024)), "Safely shut down node when free space remaining in the chain state database drops below this size (in MiB).") + ("backing-store", boost::program_options::value()->default_value(eosio::chain::backing_store_type::CHAINBASE), + "The storage for state, chainbase or rocksdb") + ("persistent-storage-num-threads", bpo::value()->default_value(default_persistent_storage_num_threads), + "Number of rocksdb threads for flush and compaction") + ("persistent-storage-max-num-files", bpo::value()->default_value(config::default_persistent_storage_max_num_files), + "Max number of rocksdb files to keep open. -1 = unlimited.") + ("persistent-storage-write-buffer-size-mb", bpo::value()->default_value(config::default_persistent_storage_write_buffer_size / (1024 * 1024)), + "Size of a single rocksdb memtable (in MiB)") + ("persistent-storage-bytes-per-sync", bpo::value()->default_value(config::default_persistent_storage_bytes_per_sync), + "Rocksdb write rate of flushes and compactions.") + ("persistent-storage-mbytes-snapshot-batch", bpo::value()->default_value(config::default_persistent_storage_mbytes_batch), + "Rocksdb batch size threshold before writing read in snapshot data to database.") + ("reversible-blocks-db-size-mb", bpo::value()->default_value(config::default_reversible_cache_size / (1024 * 1024)), "Maximum size (in MiB) of the reversible blocks database") ("reversible-blocks-db-guard-size-mb", bpo::value()->default_value(config::default_reversible_guard_size / (1024 * 1024)), "Safely shut down node when free space remaining in the reverseible blocks database drops below this size (in MiB).") ("signature-cpu-billable-pct", bpo::value()->default_value(config::default_sig_cpu_bill_pct / config::percent_1), @@ -256,6 +334,14 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip "Number of worker threads in controller thread pool") ("contracts-console", bpo::bool_switch()->default_value(false), "print contract's output to console") + ("deep-mind", bpo::bool_switch()->default_value(false), + "print deeper information about chain operations") + ("telemetry-url", bpo::value(), + "Send Zipkin spans to url. e.g. http://127.0.0.1:9411/api/v2/spans" ) + ("telemetry-service-name", bpo::value()->default_value("nodeos"), + "Zipkin localEndpoint.serviceName sent with each span" ) + ("telemetry-timeout-us", bpo::value()->default_value(200000), + "Timeout for sending Zipkin span." ) ("actor-whitelist", boost::program_options::value>()->composing()->multitoken(), "Account added to actor whitelist (may specify multiple times)") ("actor-blacklist", boost::program_options::value>()->composing()->multitoken(), @@ -284,22 +370,21 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip "In \"light\" mode all incoming blocks headers will be fully validated; transactions in those validated blocks will be trusted \n") ("disable-ram-billing-notify-checks", bpo::bool_switch()->default_value(false), "Disable the check which subjectively fails a transaction if a contract bills more RAM to another account within the context of a notification handler (i.e. when the receiver is not the code of the action).") +#ifdef EOSIO_DEVELOPER + ("disable-all-subjective-mitigations", bpo::bool_switch()->default_value(false), + "Disable all subjective mitigations checks in the entire codebase.") +#endif ("maximum-variable-signature-length", bpo::value()->default_value(16384u), "Subjectively limit the maximum length of variable components in a variable legnth signature to this size in bytes") ("trusted-producer", bpo::value>()->composing(), "Indicate a producer whose blocks headers signed by it will be fully validated, but transactions in those validated blocks will be trusted.") ("database-map-mode", bpo::value()->default_value(chainbase::pinnable_mapped_file::map_mode::mapped), "Database map mode (\"mapped\", \"heap\", or \"locked\").\n" "In \"mapped\" mode database is memory mapped as a file.\n" - "In \"heap\" mode database is preloaded in to swappable memory.\n" -#ifdef __linux__ - "In \"locked\" mode database is preloaded, locked in to memory, and optionally can use huge pages.\n" -#else - "In \"locked\" mode database is preloaded and locked in to memory.\n" +#ifndef _WIN32 + "In \"heap\" mode database is preloaded in to swappable memory and will use huge pages if available.\n" + "In \"locked\" mode database is preloaded, locked in to memory, and will use huge pages if available.\n" #endif ) -#ifdef __linux__ - ("database-hugepage-path", bpo::value>()->composing(), "Optional path for database hugepages when in \"locked\" mode (may specify multiple times)") -#endif #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED ("eos-vm-oc-cache-size-mb", bpo::value()->default_value(eosvmoc::config().cache_size / (1024u*1024u)), "Maximum size (in MiB) of the EOS VM OC code cache") @@ -339,7 +424,7 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip ("fix-reversible-blocks", bpo::bool_switch()->default_value(false), "recovers reversible block database if that database is in a bad state") ("force-all-checks", bpo::bool_switch()->default_value(false), - "do not skip any checks that can be skipped while replaying irreversible blocks") + "do not skip any validation checks while replaying blocks (useful for replaying blocks from untrusted source)") ("disable-replay-opts", bpo::bool_switch()->default_value(false), "disable optimizations that specifically target replay") ("replay-blockchain", bpo::bool_switch()->default_value(false), @@ -350,6 +435,8 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip "clear chain state database and block log") ("truncate-at-block", bpo::value()->default_value(0), "stop hard replay / block log recovery at this block number (if set to non-zero number)") + ("terminate-at-block", bpo::value()->default_value(0), + "terminate after reaching this block number (if set to a non-zero number)") ("import-reversible-blocks", bpo::value(), "replace reversible block database with blocks imported from specified file and then exit") ("export-reversible-blocks", bpo::value(), @@ -398,15 +485,7 @@ void clear_directory_contents( const fc::path& p ) { } } -void clear_chainbase_files( const fc::path& p ) { - if( !fc::is_directory( p ) ) - return; - - fc::remove( p / "shared_memory.bin" ); - fc::remove( p / "shared_memory.meta" ); -} - -optional read_builtin_protocol_feature( const fc::path& p ) { +std::optional read_builtin_protocol_feature( const fc::path& p ) { try { return fc::json::from_file( p ); } catch( const fc::exception& e ) { @@ -478,7 +557,7 @@ protocol_feature_set initialize_protocol_features( const fc::path& p, bool popul map found_builtin_protocol_features; map > builtin_protocol_features_to_add; // The bool in the pair is set to true if the builtin protocol feature has already been visited to add - map< builtin_protocol_feature_t, optional > visited_builtins; + map< builtin_protocol_feature_t, std::optional > visited_builtins; // Read all builtin protocol features if( directory_exists ) { @@ -568,7 +647,7 @@ protocol_feature_set initialize_protocol_features( const fc::path& p, bool popul std::function add_missing_builtins = [&pfs, &visited_builtins, &output_protocol_feature, &log_recognized_protocol_feature, &add_missing_builtins, populate_missing_builtins] ( builtin_protocol_feature_t codename ) -> digest_type { - auto res = visited_builtins.emplace( codename, optional() ); + auto res = visited_builtins.emplace( codename, std::optional() ); if( !res.second ) { EOS_ASSERT( res.first->second, protocol_feature_exception, "invariant failure: cycle found in builtin protocol feature dependencies" @@ -609,19 +688,19 @@ void chain_plugin::do_hard_replay(const variables_map& options) { ilog( "Hard replay requested: deleting state database" ); clear_directory_contents( my->chain_config->state_dir ); - auto backup_dir = block_log::repair_log( my->blocks_dir, options.at( "truncate-at-block" ).as()); + auto backup_dir = block_log::repair_log( my->blocks_dir, options.at( "truncate-at-block" ).as(),config::reversible_blocks_dir_name); if( fc::exists( backup_dir / config::reversible_blocks_dir_name ) || options.at( "fix-reversible-blocks" ).as()) { // Do not try to recover reversible blocks if the directory does not exist, unless the option was explicitly provided. if( !recover_reversible_blocks( backup_dir / config::reversible_blocks_dir_name, my->chain_config->reversible_cache_size, - my->chain_config->blocks_dir / config::reversible_blocks_dir_name, + my->chain_config->blog.log_dir / config::reversible_blocks_dir_name, options.at( "truncate-at-block" ).as())) { ilog( "Reversible blocks database was not corrupted. Copying from backup to blocks directory." ); fc::copy( backup_dir / config::reversible_blocks_dir_name, - my->chain_config->blocks_dir / config::reversible_blocks_dir_name ); + my->chain_config->blog.log_dir / config::reversible_blocks_dir_name ); fc::copy( backup_dir / config::reversible_blocks_dir_name / "shared_memory.bin", - my->chain_config->blocks_dir / config::reversible_blocks_dir_name / "shared_memory.bin" ); + my->chain_config->blog.log_dir / config::reversible_blocks_dir_name / "shared_memory.bin" ); } } } @@ -632,7 +711,7 @@ void chain_plugin::plugin_initialize(const variables_map& options) { try { try { genesis_state gs; // Check if EOSIO_ROOT_KEY is bad - } catch ( const fc::exception& ) { + } catch ( const std::exception& ) { elog( "EOSIO_ROOT_KEY ('${root_key}') is invalid. Recompile with a valid public key.", ("root_key", genesis_state::eosio_root_key)); throw; @@ -731,12 +810,24 @@ void chain_plugin::plugin_initialize(const variables_map& options) { if( options.count( "wasm-runtime" )) my->wasm_runtime = options.at( "wasm-runtime" ).as(); - if(options.count("abi-serializer-max-time-ms")) + if(options.count("abi-serializer-max-time-ms")) { my->abi_serializer_max_time_us = fc::microseconds(options.at("abi-serializer-max-time-ms").as() * 1000); + my->chain_config->abi_serializer_max_time_us = my->abi_serializer_max_time_us; + } - my->chain_config->blocks_dir = my->blocks_dir; - my->chain_config->state_dir = app().data_dir() / config::default_state_dir_name; - my->chain_config->read_only = my->readonly; + my->chain_config->blog.log_dir = my->blocks_dir; + my->chain_config->state_dir = app().data_dir() / config::default_state_dir_name; + my->chain_config->read_only = my->readonly; + my->chain_config->blog.retained_dir = options.at("blocks-retained-dir").as(); + my->chain_config->blog.archive_dir = options.at("blocks-archive-dir").as(); + my->chain_config->blog.stride = options.at("blocks-log-stride").as(); + my->chain_config->blog.max_retained_files = options.at("max-retained-block-files").as(); + my->chain_config->blog.fix_irreversible_blocks = options.at("fix-irreversible-blocks").as(); + + if (auto resmon_plugin = app().find_plugin()) { + resmon_plugin->monitor_directory(my->chain_config->blog.log_dir); + resmon_plugin->monitor_directory(my->chain_config->state_dir); + } if( options.count( "chain-state-db-size-mb" )) my->chain_config->state_size = options.at( "chain-state-db-size-mb" ).as() * 1024 * 1024; @@ -744,6 +835,36 @@ void chain_plugin::plugin_initialize(const variables_map& options) { if( options.count( "chain-state-db-guard-size-mb" )) my->chain_config->state_guard_size = options.at( "chain-state-db-guard-size-mb" ).as() * 1024 * 1024; + my->chain_config->backing_store = options.at( "backing-store" ).as(); + + if( options.count( "persistent-storage-num-threads" )) { + my->chain_config->persistent_storage_num_threads = options.at( "persistent-storage-num-threads" ).as(); + EOS_ASSERT( my->chain_config->persistent_storage_num_threads > 0, plugin_config_exception, + "persistent-storage-num-threads ${num} must be greater than 0", ("num", my->chain_config->persistent_storage_num_threads) ); + } + + if( options.count( "persistent-storage-max-num-files" )) { + my->chain_config->persistent_storage_max_num_files = options.at( "persistent-storage-max-num-files" ).as(); + EOS_ASSERT( my->chain_config->persistent_storage_max_num_files == -1 || my->chain_config->persistent_storage_max_num_files > 0, plugin_config_exception, + "persistent-storage-max-num-files ${num} must be equal to -1 or be greater than 0", ("num", my->chain_config->persistent_storage_max_num_files) ); + } + + if( options.count( "persistent-storage-write-buffer-size-mb" )) { + my->chain_config->persistent_storage_write_buffer_size = options.at( "persistent-storage-write-buffer-size-mb" ).as() * 1024 * 1024; + EOS_ASSERT( my->chain_config->persistent_storage_write_buffer_size > 0, plugin_config_exception, + "persistent-storage-write-buffer-size-mb ${num} must be greater than 0", ("num", my->chain_config->persistent_storage_write_buffer_size) ); + } + + if( options.count( "persistent-storage-bytes-per-sync" )) { + my->chain_config->persistent_storage_bytes_per_sync = options.at( "persistent-storage-bytes-per-sync" ).as(); + EOS_ASSERT( my->chain_config->persistent_storage_bytes_per_sync > 0, plugin_config_exception, + "persistent-storage-bytes-per-sync ${num} must be greater than 0", ("num", my->chain_config->persistent_storage_bytes_per_sync) ); + } + + my->chain_config->persistent_storage_mbytes_batch = options.at( "persistent-storage-mbytes-snapshot-batch" ).as(); + EOS_ASSERT( my->chain_config->persistent_storage_mbytes_batch > 0, plugin_config_exception, + "persistent-storage-mbytes-snapshot-batch ${num} must be greater than 0", ("num", my->chain_config->persistent_storage_mbytes_batch) ); + if( options.count( "reversible-blocks-db-size-mb" )) my->chain_config->reversible_cache_size = options.at( "reversible-blocks-db-size-mb" ).as() * 1024 * 1024; @@ -772,10 +893,18 @@ void chain_plugin::plugin_initialize(const variables_map& options) { my->chain_config->disable_replay_opts = options.at( "disable-replay-opts" ).as(); my->chain_config->contracts_console = options.at( "contracts-console" ).as(); my->chain_config->allow_ram_billing_in_notify = options.at( "disable-ram-billing-notify-checks" ).as(); + +#ifdef EOSIO_DEVELOPER + my->chain_config->disable_all_subjective_mitigations = options.at( "disable-all-subjective-mitigations" ).as(); +#endif + my->chain_config->maximum_variable_signature_length = options.at( "maximum-variable-signature-length" ).as(); + if( options.count( "terminate-at-block" )) + my->chain_config->terminate_at_block = options.at( "terminate-at-block" ).as(); + if( options.count( "extract-genesis-json" ) || options.at( "print-genesis-json" ).as()) { - fc::optional gs; + std::optional gs; if( fc::exists( my->blocks_dir / "blocks.log" )) { gs = block_log::extract_genesis_state( my->blocks_dir ); @@ -820,7 +949,7 @@ void chain_plugin::plugin_initialize(const variables_map& options) { p = bfs::current_path() / p; } - if( export_reversible_blocks( my->chain_config->blocks_dir/config::reversible_blocks_dir_name, p ) ) + if( export_reversible_blocks( my->chain_config->blog.log_dir/config::reversible_blocks_dir_name, p ) ) ilog( "Saved all blocks from reversible block database into '${path}'", ("path", p.generic_string()) ); else ilog( "Saved recovered blocks from reversible block database into '${path}'", ("path", p.generic_string()) ); @@ -840,17 +969,17 @@ void chain_plugin::plugin_initialize(const variables_map& options) { ilog( "Replay requested: deleting state database" ); if( options.at( "truncate-at-block" ).as() > 0 ) wlog( "The --truncate-at-block option does not work for a regular replay of the blockchain." ); - clear_chainbase_files( my->chain_config->state_dir ); + eosio::chain::combined_database::destroy( my->chain_config->state_dir ); if( options.at( "fix-reversible-blocks" ).as()) { - if( !recover_reversible_blocks( my->chain_config->blocks_dir / config::reversible_blocks_dir_name, + if( !recover_reversible_blocks( my->chain_config->blog.log_dir / config::reversible_blocks_dir_name, my->chain_config->reversible_cache_size )) { ilog( "Reversible blocks database was not corrupted." ); } } } else if( options.at( "fix-reversible-blocks" ).as()) { - if( !recover_reversible_blocks( my->chain_config->blocks_dir / config::reversible_blocks_dir_name, + if( !recover_reversible_blocks( my->chain_config->blog.log_dir / config::reversible_blocks_dir_name, my->chain_config->reversible_cache_size, - optional(), + std::optional(), options.at( "truncate-at-block" ).as())) { ilog( "Reversible blocks database verified to not be corrupted. Now exiting..." ); } else { @@ -862,9 +991,9 @@ void chain_plugin::plugin_initialize(const variables_map& options) { } else if( options.count("import-reversible-blocks") ) { auto reversible_blocks_file = options.at("import-reversible-blocks").as(); ilog("Importing reversible blocks from '${file}'", ("file", reversible_blocks_file.generic_string()) ); - fc::remove_all( my->chain_config->blocks_dir/config::reversible_blocks_dir_name ); + fc::remove_all( my->chain_config->blog.log_dir/config::reversible_blocks_dir_name ); - import_reversible_blocks( my->chain_config->blocks_dir/config::reversible_blocks_dir_name, + import_reversible_blocks( my->chain_config->blog.log_dir/config::reversible_blocks_dir_name, my->chain_config->reversible_cache_size, reversible_blocks_file ); EOS_THROW( node_management_success, "imported reversible blocks" ); @@ -874,7 +1003,7 @@ void chain_plugin::plugin_initialize(const variables_map& options) { wlog("The --import-reversible-blocks option should be used by itself."); } - fc::optional chain_id; + std::optional chain_id; if (options.count( "snapshot" )) { my->snapshot_path = options.at( "snapshot" ).as(); EOS_ASSERT( fc::exists(*my->snapshot_path), plugin_config_exception, @@ -925,8 +1054,8 @@ void chain_plugin::plugin_initialize(const variables_map& options) { chain_id = controller::extract_chain_id_from_db( my->chain_config->state_dir ); - fc::optional block_log_genesis; - fc::optional block_log_chain_id; + std::optional block_log_genesis; + std::optional block_log_chain_id; if( fc::is_regular_file( my->blocks_dir / "blocks.log" ) ) { block_log_genesis = block_log::extract_genesis_state( my->blocks_dir ); @@ -1055,10 +1184,6 @@ void chain_plugin::plugin_initialize(const variables_map& options) { } my->chain_config->db_map_mode = options.at("database-map-mode").as(); -#ifdef __linux__ - if( options.count("database-hugepage-path") ) - my->chain_config->db_hugepage_paths = options.at("database-hugepage-path").as>(); -#endif #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED if( options.count("eos-vm-oc-cache-size-mb") ) @@ -1073,6 +1198,42 @@ void chain_plugin::plugin_initialize(const variables_map& options) { my->chain.emplace( *my->chain_config, std::move(pfs), *chain_id ); + // initialize deep mind logging + if ( options.at( "deep-mind" ).as() ) { + // The actual `fc::dmlog_appender` implementation that is currently used by deep mind + // logger is using `stdout` to prints it's log line out. Deep mind logging outputs + // massive amount of data out of the process, which can lead under pressure to some + // of the system calls (i.e. `fwrite`) to fail abruptly without fully writing the + // entire line. + // + // Recovering from errors on a buffered (line or full) and continuing retrying write + // is merely impossible to do right, because the buffer is actually held by the + // underlying `libc` implementation nor the operation system. + // + // To ensure good functionalities of deep mind tracer, the `stdout` is made unbuffered + // and the actual `fc::dmlog_appender` deals with retry when facing error, enabling a much + // more robust deep mind output. + // + // Changing the standard `stdout` behavior from buffered to unbuffered can is disruptive + // and can lead to weird scenarios in the logging process if `stdout` is used there too. + // + // In a future version, the `fc::dmlog_appender` implementation will switch to a `FIFO` file + // approach, which will remove the dependency on `stdout` and hence this call. + // + // For the time being, when `deep-mind = true` is activated, we set `stdout` here to + // be an unbuffered I/O stream. + setbuf(stdout, NULL); + + my->chain->enable_deep_mind( &_deep_mind_log ); + } + + if (options.count("telemetry-url")) { + fc::zipkin_config::init( options["telemetry-url"].as(), + options["telemetry-service-name"].as(), + options["telemetry-timeout-us"].as() ); + } + + // set up method providers my->get_block_by_number_provider = app().get_method().register_provider( [this]( uint32_t block_num ) -> signed_block_ptr { @@ -1097,7 +1258,7 @@ void chain_plugin::plugin_initialize(const variables_map& options) { my->pre_accepted_block_connection = my->chain->pre_accepted_block.connect([this](const signed_block_ptr& blk) { auto itr = my->loaded_checkpoints.find( blk->block_num() ); if( itr != my->loaded_checkpoints.end() ) { - auto id = blk->id(); + auto id = blk->calculate_id(); EOS_ASSERT( itr->second == id, checkpoint_exception, "Checkpoint does not match for block number ${num}: expected: ${expected} actual: ${actual}", ("num", blk->block_num())("expected", itr->second)("actual", id) @@ -1113,9 +1274,19 @@ void chain_plugin::plugin_initialize(const variables_map& options) { } ); my->accepted_block_connection = my->chain->accepted_block.connect( [this]( const block_state_ptr& blk ) { - if (my->_account_query_db) { - my->_account_query_db->commit_block(blk); + if (auto dm_logger = my->chain->get_deep_mind_logger()) { + auto packed_blk = fc::raw::pack(*blk); + + fc_dlog(*dm_logger, "ACCEPTED_BLOCK ${num} ${blk}", + ("num", blk->block_num) + ("blk", fc::to_hex(packed_blk)) + ); } + + if (my->_account_query_db) { + my->_account_query_db->commit_block(blk); + } + my->accepted_block_channel.publish( priority::high, blk ); } ); @@ -1129,10 +1300,20 @@ void chain_plugin::plugin_initialize(const variables_map& options) { } ); my->applied_transaction_connection = my->chain->applied_transaction.connect( - [this]( std::tuple t ) { + [this]( std::tuple t ) { + if (auto dm_logger = my->chain->get_deep_mind_logger()) { + auto packed_trace = fc::raw::pack(*std::get<0>(t)); + + fc_dlog(*dm_logger, "APPLIED_TRANSACTION ${block} ${traces}", + ("block", my->chain->head_block_num() + 1) + ("traces", fc::to_hex(packed_trace)) + ); + } + if (my->_account_query_db) { my->_account_query_db->cache_transaction_trace(std::get<0>(t)); } + my->applied_transaction_channel.publish( priority::low, std::get<0>(t) ); } ); @@ -1143,22 +1324,25 @@ void chain_plugin::plugin_initialize(const variables_map& options) { void chain_plugin::plugin_startup() { try { + handle_sighup(); // Sets loggers + EOS_ASSERT( my->chain_config->read_mode != db_read_mode::IRREVERSIBLE || !accept_transactions(), plugin_config_exception, "read-mode = irreversible. transactions should not be enabled by enable_accept_transactions" ); try { - my->producer_plug = app().find_plugin(); - EOS_ASSERT(my->producer_plug, plugin_exception, "Failed to find producer_plugin"); - - auto shutdown = [](){ return app().is_quiting(); }; - if (my->snapshot_path) { + auto shutdown = [](){ return app().quit(); }; + auto check_shutdown = [](){ return app().is_quiting(); }; + auto bvc_plug = app().find_plugin(); + auto blockvault_instance = bvc_plug ? bvc_plug->get() : nullptr; + if (nullptr != blockvault_instance) { + eosio::blockvault::blockvault_sync_strategy bss(blockvault_instance, *my, shutdown, check_shutdown); + bss.do_sync(); + } else if (my->snapshot_path) { auto infile = std::ifstream(my->snapshot_path->generic_string(), (std::ios::in | std::ios::binary)); auto reader = std::make_shared(infile); - my->chain->startup(shutdown, reader); + my->chain->startup(shutdown, check_shutdown, reader); infile.close(); - } else if( my->genesis ) { - my->chain->startup(shutdown, *my->genesis); } else { - my->chain->startup(shutdown); + my->do_non_snapshot_startup(shutdown, check_shutdown); } } catch (const database_guard_exception& e) { log_guard_exception(e); @@ -1180,7 +1364,7 @@ void chain_plugin::plugin_startup() } my->chain_config.reset(); - + if (my->account_queries_enabled) { my->account_queries_enabled = false; try { @@ -1190,6 +1374,7 @@ void chain_plugin::plugin_startup() } + } FC_CAPTURE_AND_RETHROW() } void chain_plugin::plugin_shutdown() { @@ -1202,6 +1387,11 @@ void chain_plugin::plugin_shutdown() { if(app().is_quiting()) my->chain->get_wasm_interface().indicate_shutting_down(); my->chain.reset(); + zipkin_config::shutdown(); +} + +void chain_plugin::handle_sighup() { + fc::logger::update( deep_mind_logger_name, _deep_mind_log ); } chain_apis::read_write::read_write(controller& db, const fc::microseconds& abi_serializer_max_time, bool api_accept_transactions) @@ -1217,10 +1407,10 @@ void chain_apis::read_write::validate() const { } chain_apis::read_only chain_plugin::get_read_only_api() const { - return chain_apis::read_only(chain(), my->_account_query_db, get_abi_serializer_max_time(), my->producer_plug); + return chain_apis::read_only(chain(), my->_account_query_db, get_abi_serializer_max_time()); } - + bool chain_plugin::accept_block(const signed_block_ptr& block, const block_id_type& id ) { return my->incoming_block_sync_method(block, id); } @@ -1229,13 +1419,8 @@ void chain_plugin::accept_transaction(const chain::packed_transaction_ptr& trx, my->incoming_transaction_async_method(trx, false, std::move(next)); } -bool chain_plugin::block_is_on_preferred_chain(const block_id_type& block_id) { - auto b = chain().fetch_block_by_number( block_header::num_from_id(block_id) ); - return b && b->id() == block_id; -} - bool chain_plugin::recover_reversible_blocks( const fc::path& db_dir, uint32_t cache_size, - optional new_db_dir, uint32_t truncate_at_block ) { + std::optional new_db_dir, uint32_t truncate_at_block ) { try { chainbase::database reversible( db_dir, database::read_only); // Test if dirty // If it reaches here, then the reversible database is not dirty @@ -1284,7 +1469,7 @@ bool chain_plugin::recover_reversible_blocks( const fc::path& db_dir, uint32_t c ilog( "Reconstructing '${reversible_dir}' from backed up reversible directory", ("reversible_dir", reversible_dir) ); - optional old_reversible; + std::optional old_reversible; try { old_reversible = chainbase::database( backup_dir, database::read_only, 0, true ); @@ -1365,7 +1550,7 @@ bool chain_plugin::import_reversible_blocks( const fc::path& reversible_dir, new_reversible.add_index(); try { while( reversible_blocks.tellg() < end_pos ) { - signed_block tmp; + signed_block_v0 tmp; fc::raw::unpack(reversible_blocks, tmp); num = tmp.block_num(); @@ -1380,7 +1565,7 @@ bool chain_plugin::import_reversible_blocks( const fc::path& reversible_dir, new_reversible.create( [&]( auto& ubo ) { ubo.blocknum = num; - ubo.set_block( std::make_shared(std::move(tmp)) ); + ubo.set_block( std::make_shared(std::move(tmp), true) ); }); end = num; } @@ -1422,7 +1607,9 @@ bool chain_plugin::export_reversible_blocks( const fc::path& reversible_dir, signed_block tmp; fc::datastream ds( itr->packedblock.data(), itr->packedblock.size() ); fc::raw::unpack(ds, tmp); // Verify that packed block has not been corrupted. - reversible_blocks.write( itr->packedblock.data(), itr->packedblock.size() ); + const auto v0 = tmp.to_signed_block_v0(); // store in signed_block_v0 format + auto packed_v0 = fc::raw::pack(*v0); + reversible_blocks.write( packed_v0.data(), packed_v0.size() ); end = itr->blocknum; ++num; } @@ -1482,7 +1669,7 @@ void chain_plugin::log_guard_exception(const chain::guard_exception&e ) { void chain_plugin::handle_guard_exception(const chain::guard_exception& e) { log_guard_exception(e); - elog("database chain::guard_exception, quiting..."); // log string searched for in: tests/nodeos_under_min_avail_ram.py + elog("database chain::guard_exception, quitting..."); // log string searched for in: tests/nodeos_under_min_avail_ram.py // quit the app app().quit(); } @@ -1498,7 +1685,7 @@ void chain_plugin::handle_bad_alloc() { //return -2 -- it's what programs/nodeos/main.cpp reports for std::exception std::_Exit(-2); } - + bool chain_plugin::account_queries_enabled() const { return my->account_queries_enabled; } @@ -1537,7 +1724,8 @@ read_only::get_info_results read_only::get_info(const read_only::get_info_params app().version_string(), db.fork_db_pending_head_block_num(), db.fork_db_pending_head_block_id(), - app().full_version_string() + app().full_version_string(), + db.last_irreversible_block_time() }; } @@ -1654,6 +1842,10 @@ uint64_t read_only::get_table_index_name(const read_only::get_table_rows_params& return index; } +uint64_t convert_to_type(const eosio::name &n, const string &desc) { + return n.to_uint64_t(); +} + template<> uint64_t convert_to_type(const string& str, const string& desc) { @@ -1818,11 +2010,504 @@ read_only::get_table_rows_result read_only::get_table_rows( const read_only::get #pragma GCC diagnostic pop } +/// short_string is intended to optimize the string equality comparison where one of the operand is +/// no greater than 8 bytes long. +struct short_string { + uint64_t data = 0; + + template + short_string(const char (&str)[SIZE]) { + static_assert(SIZE <= 8, "data has to be 8 bytes or less"); + memcpy(&data, str, SIZE); + } + + short_string(std::string str) { memcpy(&data, str.c_str(), std::min(sizeof(data), str.size())); } + + bool empty() const { return data == 0; } + + friend bool operator==(short_string lhs, short_string rhs) { return lhs.data == rhs.data; } + friend bool operator!=(short_string lhs, short_string rhs) { return lhs.data != rhs.data; } +}; + +template +struct key_converter; + +inline void key_convert_assert(bool condition) { + // EOS_ASSERT is avoided intentionally here because EOS_ASSERT would create the fc::log_message object which is + // relatively expensive. The throw statement here is only used for flow control purpose, not for error reporting + // purpose. + if (!condition) + throw std::invalid_argument(""); +} + +// convert unsigned integer in hex representation back to its integer representation +template +UnsignedInt unhex(const std::string& bytes_in_hex) { + assert(bytes_in_hex.size() == 2 * sizeof(UnsignedInt)); + std::array bytes; + boost::algorithm::unhex(bytes_in_hex.begin(), bytes_in_hex.end(), bytes.rbegin()); + UnsignedInt result; + memcpy(&result, bytes.data(), sizeof(result)); + return result; +} + +template +struct key_converter>> { + static void to_bytes(const string& str, short_string encode_type, fixed_buf_stream& strm) { + int base = 10; + if (encode_type == "hex") + base = 16; + else + key_convert_assert(encode_type.empty() || encode_type == "dec"); + + size_t pos = 0; + if constexpr (std::is_unsigned_v) { + uint64_t value = std::stoul(str, &pos, base); + key_convert_assert(pos > 0 && value <= std::numeric_limits::max()); + to_key(static_cast(value), strm); + } else { + int64_t value = std::stol(str, &pos, base); + key_convert_assert(pos > 0 && value <= std::numeric_limits::max() && + value >= std::numeric_limits::min()); + to_key(static_cast(value), strm); + } + } + + static IntType value_from_hex(const std::string& bytes_in_hex) { + auto unsigned_val = unhex>(bytes_in_hex); + if (unsigned_val > std::numeric_limits::max()) { + return unsigned_val + static_cast>(std::numeric_limits::min()); + } else { + return unsigned_val + std::numeric_limits::min(); + } + } + + static std::string from_hex(const std::string& bytes_in_hex, short_string encode_type) { + IntType val = value_from_hex(bytes_in_hex); + if (encode_type.empty() || encode_type == "dec") { + return std::to_string(val); + } else if (encode_type == "hex") { + std::array v; + memcpy(v.data(), &val, sizeof(val)); + char result[2 * sizeof(IntType) + 1] = {'\0'}; + boost::algorithm::hex(v.rbegin(), v.rend(), result); + return std::find_if_not(result, result + 2 * sizeof(IntType), [](char v) { return v == '0'; }); + } + throw std::invalid_argument(""); + } +}; + +template +struct key_converter>> { + static void to_bytes(const string& str, short_string encode_type, fixed_buf_stream& strm) { + key_convert_assert(encode_type.empty() || encode_type == "dec"); + if constexpr (sizeof(Float) == 4) { + to_key(std::stof(str), strm); + } else { + to_key(std::stod(str), strm); + } + } + + static Float value_from_hex(const std::string& bytes_in_hex) { + using UInt = std::conditional_t; + UInt val = unhex(bytes_in_hex); + + UInt mask = 0; + UInt signbit = (static_cast(1) << (std::numeric_limits::digits - 1)); + if (!(val & signbit)) // flip mask if val is positive + mask = ~mask; + val ^= (mask | signbit); + Float result; + memcpy(&result, &val, sizeof(val)); + return result; + } + + static std::string from_hex(const std::string& bytes_in_hex, short_string encode_type) { + return std::to_string(value_from_hex(bytes_in_hex)); + } +}; + +template <> +struct key_converter { + static void to_bytes(const string& str, short_string encode_type, fixed_buf_stream& strm) { + key_convert_assert(encode_type.empty() || encode_type == "hex"); + checksum256_type sha{str}; + strm.write(sha.data(), sha.data_size()); + } + static std::string from_hex(const std::string& bytes_in_hex, short_string encode_type) { return bytes_in_hex; } +}; + +template <> +struct key_converter { + static void to_bytes(const string& str, short_string encode_type, fixed_buf_stream& strm) { + key_convert_assert(encode_type.empty() || encode_type == "name"); + to_key(name(str).to_uint64_t(), strm); + } + + static std::string from_hex(const std::string& bytes_in_hex, short_string encode_type) { + return name(key_converter::value_from_hex(bytes_in_hex)).to_string(); + } +}; + +template <> +struct key_converter { + static void to_bytes(const string& str, short_string encode_type, fixed_buf_stream& strm) { + key_convert_assert(encode_type.empty() || encode_type == "string"); + to_key(str, strm); + } + + static std::string from_hex(const std::string& bytes_in_hex, short_string encode_type) { + std::string result = boost::algorithm::unhex(bytes_in_hex); + /// restore the string following the encoding rule from `template to_key(std::string, S&)` in abieos + /// to_key.hpp + boost::replace_all(result, "\0\1", "\0"); + // remove trailing '\0\0' + auto sz = result.size(); + if (sz >= 2 && result[sz - 1] == '\0' && result[sz - 2] == '\0') + result.resize(sz - 2); + return result; + } +}; + +namespace key_helper { +/// Caution: the order of `key_type` and `key_type_ids` should match exactly. +using key_types = std::tuple; +static const short_string key_type_ids[] = {"int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", + "uint64", "float32", "float64", "name", "sha256", "i256", "string"}; + +static_assert(sizeof(key_type_ids) / sizeof(short_string) == std::tuple_size::value, + "key_type_ids and key_types must be of the same size and the order of their elements has to match"); + +uint64_t type_string_to_function_index(short_string name) { + unsigned index = std::find(std::begin(key_type_ids), std::end(key_type_ids), name) - std::begin(key_type_ids); + key_convert_assert(index < std::tuple_size::value); + return index; +} + +void write_key(string index_type, string encode_type, const string& index_value, fixed_buf_stream& strm) { + try { + // converts arbitrary hex strings to bytes ex) "FFFEFD" to {255, 254, 253} + if (encode_type == "bytes") { + strm.pos = boost::algorithm::unhex(index_value.begin(), index_value.end(), strm.pos); + return; + } + + if (index_type == "ripemd160") { + key_convert_assert(encode_type.empty() || encode_type == "hex"); + checksum160_type ripem160{index_value}; + strm.write(ripem160.data(), ripem160.data_size()); + return; + } + + std::apply( + [index_type, &index_value, encode_type, &strm](auto... t) { + using to_byte_fun_t = void (*)(const string&, short_string, fixed_buf_stream&); + static to_byte_fun_t funs[] = {&key_converter::to_bytes...}; + auto index = type_string_to_function_index(index_type); + funs[index](index_value, encode_type, strm); + }, + key_types{}); + } catch (...) { + FC_THROW_EXCEPTION(chain::contract_table_query_exception, + "Incompatible index type/encode_type/Index_value: ${t}/${e}/{$v} ", + ("t", index_type)("e", encode_type)("v", index_value)); + } +} + +std::string read_key(string index_type, string encode_type, const string& bytes_in_hex) { + try { + if (encode_type == "bytes" || index_type == "ripemd160") + return bytes_in_hex; + + return std::apply( + [index_type, bytes_in_hex, &encode_type](auto... t) { + using from_hex_fun_t = std::string (*)(const string&, short_string); + static from_hex_fun_t funs[] = {&key_converter::from_hex...}; + auto index = type_string_to_function_index(index_type); + return funs[index](bytes_in_hex, encode_type); + }, + key_types{}); + } catch (...) { + FC_THROW_EXCEPTION(chain::contract_table_query_exception, "Unsupported index type/encode_type: ${t}/${e} ", + ("t", index_type)("e", encode_type)); + } +} +} // namespace key_helper + +constexpr uint32_t prefix_size = 17; // prefix 17bytes: status(1 byte) + table_name(8bytes) + index_name(8 bytes) +struct kv_table_rows_context { + std::unique_ptr kv_context; + const read_only::get_kv_table_rows_params& p; + abi_serializer::yield_function_t yield_function; + abi_def abi; + abi_serializer abis; + std::string index_type; + bool shorten_abi_errors; + bool is_primary_idx; + + kv_table_rows_context(const controller& db, const read_only::get_kv_table_rows_params& param, + const fc::microseconds abi_serializer_max_time, bool shorten_error) + : kv_context(db.kv_db().create_kv_context( + param.code, {}, + db.get_global_properties().kv_configuration)) // To do: provide kv_resource_manmager to create_kv_context + , p(param) + , yield_function(abi_serializer::create_yield_function(abi_serializer_max_time)) + , abi(eosio::chain_apis::get_abi(db, param.code)) + , shorten_abi_errors(shorten_error) { + + EOS_ASSERT(p.limit > 0, chain::contract_table_query_exception, "invalid limit : ${n}", ("n", p.limit)); + string tbl_name = p.table.to_string(); + + // Check valid table name + const auto table_it = abi.kv_tables.value.find(p.table); + if (table_it == abi.kv_tables.value.end()) { + EOS_ASSERT(false, chain::contract_table_query_exception, "Unknown kv_table: ${t}", ("t", tbl_name)); + } + const auto& kv_tbl_def = table_it->second; + // Check valid index_name + is_primary_idx = (p.index_name == kv_tbl_def.primary_index.name); + bool is_sec_idx = (kv_tbl_def.secondary_indices.find(p.index_name) != kv_tbl_def.secondary_indices.end()); + EOS_ASSERT(is_primary_idx || is_sec_idx, chain::contract_table_query_exception, "Unknown kv index: ${t} ${i}", + ("t", p.table)("i", p.index_name)); + + index_type = kv_tbl_def.get_index_type(p.index_name.to_string()); + abis.set_abi(abi, yield_function); + } + + bool point_query() const { return p.index_value.size(); } + + void write_prefix(fixed_buf_stream& strm) const { + strm.write('\1'); + to_key(p.table.to_uint64_t(), strm); + to_key(p.index_name.to_uint64_t(), strm); + } + + std::vector get_full_key(string key) const { + // the max possible encoded_key_byte_count occurs when the encoded type is string and when all characters + // in the string is '\0' + const size_t max_encoded_key_byte_count = std::max(sizeof(uint64_t), 2 * key.size() + 1); + std::vector full_key(prefix_size + max_encoded_key_byte_count); + fixed_buf_stream strm(full_key.data(), full_key.size()); + write_prefix(strm); + if (key.size()) + key_helper::write_key(index_type, p.encode_type, key, strm); + full_key.resize(strm.pos - full_key.data()); + return full_key; + } +}; + +struct kv_iterator_ex { + uint32_t key_size = 0; + uint32_t value_size = 0; + const kv_table_rows_context& context; + std::unique_ptr base; + kv_it_stat status; + + kv_iterator_ex(const kv_table_rows_context& ctx, const std::vector& full_key) + : context(ctx) { + base = context.kv_context->kv_it_create(context.p.code.to_uint64_t(), full_key.data(), prefix_size); + status = base->kv_it_lower_bound(full_key.data(), full_key.size(), &key_size, &value_size); + EOS_ASSERT(status != chain::kv_it_stat::iterator_erased, chain::contract_table_query_exception, + "Invalid iterator in ${t} ${i}", ("t", context.p.table)("i", context.p.index_name)); + } + + bool is_end() const { return status == kv_it_stat::iterator_end; } + + /// @pre ! is_end() + std::vector get_key() const { + std::vector result(key_size); + uint32_t actual_size; + base->kv_it_key(0, result.data(), key_size, actual_size); + return result; + } + + /// @pre ! is_end() + std::vector get_value() const { + std::vector result(value_size); + uint32_t actual_size; + base->kv_it_value(0, result.data(), value_size, actual_size); + if (!context.is_primary_idx) { + auto success = + context.kv_context->kv_get(context.p.code.to_uint64_t(), result.data(), result.size(), actual_size); + EOS_ASSERT(success, chain::contract_table_query_exception, "invalid secondary index in ${t} ${i}", + ("t", context.p.table)("i", context.p.index_name)); + result.resize(actual_size); + context.kv_context->kv_get_data(0, result.data(), actual_size); + } + + return result; + } + + /// @pre ! is_end() + fc::variant get_value_var() const { + std::vector row_value = get_value(); + if (context.p.json) { + try { + return context.abis.binary_to_variant(context.p.table.to_string(), row_value, + context.yield_function, + context.shorten_abi_errors); + } catch (fc::exception& e) { + } + } + return fc::variant(row_value); + } + + /// @pre ! is_end() + fc::variant get_value_and_maybe_payer_var() const { + fc::variant result = get_value_var(); + if (context.p.show_payer) { + auto maybe_payer = base->kv_it_payer(); + std::string payer = maybe_payer.has_value() ? maybe_payer.value().to_string() : ""; + return fc::mutable_variant_object("data", std::move(result))("payer", payer); + } + return result; + } + + /// @pre ! is_end() + std::string get_key_hex_string() const { + auto row_key = get_key(); + std::string result; + boost::algorithm::hex(row_key.begin() + prefix_size, row_key.end(), std::back_inserter(result)); + return result; + } + + /// @pre ! is_end() + kv_iterator_ex& operator++() { + status = base->kv_it_next(&key_size, &value_size); + return *this; + } + + /// @pre ! is_end() + kv_iterator_ex& operator--() { + status = base->kv_it_prev(&key_size, &value_size); + return *this; + } + + int key_compare(const std::vector& key) const { + return base->kv_it_key_compare(key.data(), key.size()); + } +}; + +struct kv_forward_range { + kv_iterator_ex current; + const std::vector& last_key; + + kv_forward_range(const kv_table_rows_context& ctx, const std::vector& first_key, + const std::vector& last_key) + : current(ctx, first_key) + , last_key(last_key) {} + + bool is_done() const { + return current.is_end() || + (last_key.size() > prefix_size && current.key_compare(last_key) > 0); + } + + void next() { ++current; } +}; + +struct kv_reverse_range { + kv_iterator_ex current; + const std::vector& last_key; + + kv_reverse_range(const kv_table_rows_context& ctx, const std::vector& first_key, + const std::vector& last_key) + : current(ctx, first_key) + , last_key(last_key) { + if (first_key.size() == prefix_size) { + current.status = current.base->kv_it_move_to_end(); + } + if (current.is_end() || current.key_compare(first_key) != 0) + --current; + } + + bool is_done() const { + return current.is_end() || + (last_key.size() > prefix_size && current.key_compare(last_key) < 0); + } + + void next() { --current; } +}; + +template +read_only::get_table_rows_result kv_get_rows(Range&& range) { + + keep_processing kp {}; + read_only::get_table_rows_result result; + auto& ctx = range.current.context; + for (unsigned count = 0; count < ctx.p.limit && !range.is_done() && kp() ; + ++count) { + result.rows.emplace_back(range.current.get_value_and_maybe_payer_var()); + range.next(); + } + + if (!range.is_done()) { + result.more = true; + result.next_key_bytes = range.current.get_key_hex_string(); + result.next_key = key_helper::read_key(ctx.index_type, ctx.p.encode_type, result.next_key_bytes); + } + return result; +} + +read_only::get_table_rows_result read_only::get_kv_table_rows(const read_only::get_kv_table_rows_params& p) const { + + kv_table_rows_context context{db, p, abi_serializer_max_time, shorten_abi_errors}; + + if (context.point_query()) { + EOS_ASSERT(p.lower_bound.empty() && p.upper_bound.empty(), chain::contract_table_query_exception, + "specify both index_value and ranges (i.e. lower_bound/upper_bound) is not allowed"); + read_only::get_table_rows_result result; + auto full_key = context.get_full_key(p.index_value); + kv_iterator_ex itr(context, full_key); + if (!itr.is_end() && itr.key_compare(full_key) == 0) { + result.rows.emplace_back(itr.get_value_and_maybe_payer_var()); + } + return result; + } + + auto lower_bound = context.get_full_key(p.lower_bound); + auto upper_bound = context.get_full_key(p.upper_bound); + + if (context.p.reverse == false) + return kv_get_rows(kv_forward_range(context, lower_bound, upper_bound)); + else + return kv_get_rows(kv_reverse_range(context, upper_bound, lower_bound)); +} + +struct table_receiver + : chain::backing_store::table_only_error_receiver { + table_receiver(read_only::get_table_by_scope_result& result, const read_only::get_table_by_scope_params& params) + : result_(result), params_(params) { + check_limit(); + } + + void add_table_row(const backing_store::table_id_object_view& row) { + if( params_.table && row.table != params_.table ) { + return; + } + result_.rows.push_back( {row.code, row.scope, row.table, row.payer, row.count} ); + check_limit(); + } + + auto keep_processing_entries() { + keep_processing kp {}; + return [kp{std::move(kp)},&reached_limit=reached_limit_]() { + return !reached_limit && kp(); + }; + }; + + void check_limit() { + if (result_.rows.size() >= params_.limit) + reached_limit_ = true; + } + + read_only::get_table_by_scope_result& result_; + const read_only::get_table_by_scope_params& params_; + bool reached_limit_ = false; +}; + read_only::get_table_by_scope_result read_only::get_table_by_scope( const read_only::get_table_by_scope_params& p )const { read_only::get_table_by_scope_result result; - const auto& d = db.db(); - - const auto& idx = d.get_index(); auto lower_bound_lookup_tuple = std::make_tuple( p.code, name(std::numeric_limits::lowest()), p.table ); auto upper_bound_lookup_tuple = std::make_tuple( p.code, name(std::numeric_limits::max()), (p.table.empty() ? name(std::numeric_limits::max()) : p.table) ); @@ -1840,27 +2525,60 @@ read_only::get_table_by_scope_result read_only::get_table_by_scope( const read_o if( upper_bound_lookup_tuple < lower_bound_lookup_tuple ) return result; - auto walk_table_range = [&]( auto itr, auto end_itr ) { - auto cur_time = fc::time_point::now(); - auto end_time = cur_time + fc::microseconds(1000 * 10); /// 10ms max time - for( unsigned int count = 0; cur_time <= end_time && count < p.limit && itr != end_itr; ++itr, cur_time = fc::time_point::now() ) { - if( p.table && itr->table != p.table ) continue; + const bool reverse = p.reverse && *p.reverse; + const auto db_backing_store = get_backing_store(); + if (db_backing_store == eosio::chain::backing_store_type::CHAINBASE) { + auto walk_table_range = [&result,&p]( auto itr, auto end_itr ) { + keep_processing kp; + for( unsigned int count = 0; kp() && count < p.limit && itr != end_itr; ++itr ) { + if( p.table && itr->table != p.table ) continue; - result.rows.push_back( {itr->code, itr->scope, itr->table, itr->payer, itr->count} ); + result.rows.push_back( {itr->code, itr->scope, itr->table, itr->payer, itr->count} ); - ++count; + ++count; + } + if( itr != end_itr ) { + result.more = itr->scope.to_string(); + } + }; + + const auto& d = db.db(); + const auto& idx = d.get_index(); + auto lower = idx.lower_bound( lower_bound_lookup_tuple ); + auto upper = idx.upper_bound( upper_bound_lookup_tuple ); + if( reverse ) { + walk_table_range( boost::make_reverse_iterator(upper), boost::make_reverse_iterator(lower) ); + } else { + walk_table_range( lower, upper ); } - if( itr != end_itr ) { - result.more = itr->scope.to_string(); + } + else { + using namespace eosio::chain; + EOS_ASSERT(db_backing_store == backing_store_type::ROCKSDB, + chain::contract_table_query_exception, + "Support for configured backing_store has not been added to get_table_by_scope"); + const auto& kv_database = db.kv_db(); + table_receiver receiver(result, p); + auto kp = receiver.keep_processing_entries(); + auto lower = chain::backing_store::db_key_value_format::create_full_prefix_key(std::get<0>(lower_bound_lookup_tuple), + std::get<1>(lower_bound_lookup_tuple), + std::get<2>(lower_bound_lookup_tuple)); + auto upper = chain::backing_store::db_key_value_format::create_full_prefix_key(std::get<0>(upper_bound_lookup_tuple), + std::get<1>(upper_bound_lookup_tuple), + std::get<2>(upper_bound_lookup_tuple)); + if (reverse) { + lower = eosio::session::shared_bytes::truncate_key(lower); + } + // since upper is either the upper_bound of a forward search, or the reverse iterator <= for the beginning of the end of + // the table, we need to move it to just before the beginning of the next table + upper = upper.next(); + const auto context = (reverse) ? backing_store::key_context::table_only_reverse : backing_store::key_context::table_only; + backing_store::rocksdb_contract_db_table_writer> writer(receiver, context, kp); + eosio::chain::backing_store::walk_rocksdb_entries_with_prefix(kv_database.get_kv_undo_stack(), lower, upper, writer); + const auto stopped_at = writer.stopped_at(); + if (stopped_at) { + result.more = stopped_at->scope.to_string(); } - }; - - auto lower = idx.lower_bound( lower_bound_lookup_tuple ); - auto upper = idx.upper_bound( upper_bound_lookup_tuple ); - if( p.reverse && *p.reverse ) { - walk_table_range( boost::make_reverse_iterator(upper), boost::make_reverse_iterator(lower) ); - } else { - walk_table_range( lower, upper ); } return result; @@ -1872,7 +2590,7 @@ vector read_only::get_currency_balance( const read_only::get_currency_bal (void)get_table_type( abi, name("accounts") ); vector results; - walk_key_value_table(p.code, p.account, N(accounts), [&](const key_value_object& obj){ + walk_key_value_table(p.code, p.account, "accounts"_n, [&](const auto& obj){ EOS_ASSERT( obj.value.size() >= sizeof(asset), chain::asset_type_exception, "Invalid data on table"); asset cursor; @@ -1900,7 +2618,7 @@ fc::variant read_only::get_currency_stats( const read_only::get_currency_stats_p uint64_t scope = ( eosio::chain::string_to_symbol( 0, boost::algorithm::to_upper_copy(p.symbol).c_str() ) >> 8 ); - walk_key_value_table(p.code, name(scope), N(stat), [&](const key_value_object& obj){ + walk_key_value_table(p.code, name(scope), "stat"_n, [&](const auto& obj){ EOS_ASSERT( obj.value.size() >= sizeof(read_only::get_currency_stats_result), chain::asset_type_exception, "Invalid data on table"); fc::datastream ds(obj.value.data(), obj.value.size()); @@ -1917,70 +2635,140 @@ fc::variant read_only::get_currency_stats( const read_only::get_currency_stats_p return results; } -fc::variant get_global_row( const database& db, const abi_def& abi, const abi_serializer& abis, const fc::microseconds& abi_serializer_max_time_us, bool shorten_abi_errors ) { - const auto table_type = get_table_type(abi, N(global)); - EOS_ASSERT(table_type == read_only::KEYi64, chain::contract_table_query_exception, "Invalid table type ${type} for table global", ("type",table_type)); - - const auto* const table_id = db.find(boost::make_tuple(config::system_account_name, config::system_account_name, N(global))); - EOS_ASSERT(table_id, chain::contract_table_query_exception, "Missing table global"); - - const auto& kv_index = db.get_index(); - const auto it = kv_index.find(boost::make_tuple(table_id->id, N(global).to_uint64_t())); - EOS_ASSERT(it != kv_index.end(), chain::contract_table_query_exception, "Missing row in table global"); - - vector data; - read_only::copy_inline_row(*it, data); - return abis.binary_to_variant(abis.get_table_type(N(global)), data, abi_serializer::create_yield_function( abi_serializer_max_time_us ), shorten_abi_errors ); -} - read_only::get_producers_result read_only::get_producers( const read_only::get_producers_params& p ) const try { + const auto producers_table = "producers"_n; const abi_def abi = eosio::chain_apis::get_abi(db, config::system_account_name); - const auto table_type = get_table_type(abi, N(producers)); + const auto table_type = get_table_type(abi, producers_table); const abi_serializer abis{ abi, abi_serializer::create_yield_function( abi_serializer_max_time ) }; EOS_ASSERT(table_type == KEYi64, chain::contract_table_query_exception, "Invalid table type ${type} for table producers", ("type",table_type)); const auto& d = db.db(); const auto lower = name{p.lower_bound}; + keep_processing kp; + read_only::get_producers_result result; + auto done = [&kp,&result,&limit=p.limit](const auto& row) { + if (result.rows.size() >= limit || !kp()) { + result.more = name{row.primary_key}.to_string(); + return true; + } + return false; + }; + auto type = abis.get_table_type(producers_table); + auto get_val = get_primary_key_value(type, abis, p.json); + auto add_val = [&result,get_val{std::move(get_val)}](const auto& row) { + fc::variant data_var; + get_val(data_var, row); + result.rows.emplace_back(std::move(data_var)); + }; + + const auto code = config::system_account_name; + const auto scope = config::system_account_name; static const uint8_t secondary_index_num = 0; - const auto* const table_id = d.find( - boost::make_tuple(config::system_account_name, config::system_account_name, N(producers))); - const auto* const secondary_table_id = d.find( - boost::make_tuple(config::system_account_name, config::system_account_name, name(N(producers).to_uint64_t() | secondary_index_num))); - EOS_ASSERT(table_id && secondary_table_id, chain::contract_table_query_exception, "Missing producers table"); + const name sec_producers_table {producers_table.to_uint64_t() | secondary_index_num}; + + const auto db_backing_store = get_backing_store(); + if (db_backing_store == eosio::chain::backing_store_type::CHAINBASE) { + const auto* const table_id = d.find( + boost::make_tuple(code, scope, producers_table)); + const auto* const secondary_table_id = d.find( + boost::make_tuple(code, scope, sec_producers_table)); + EOS_ASSERT(table_id && secondary_table_id, chain::contract_table_query_exception, "Missing producers table"); + + const auto& kv_index = d.get_index(); + const auto& secondary_index = d.get_index().indices(); + const auto& secondary_index_by_primary = secondary_index.get(); + const auto& secondary_index_by_secondary = secondary_index.get(); + + vector data; + + auto it = lower.to_uint64_t() == 0 + ? secondary_index_by_secondary.lower_bound( + boost::make_tuple(secondary_table_id->id, to_softfloat64(std::numeric_limits::lowest()), 0)) + : secondary_index.project( + secondary_index_by_primary.lower_bound( + boost::make_tuple(secondary_table_id->id, lower.to_uint64_t()))); + for( ; it != secondary_index_by_secondary.end() && it->t_id == secondary_table_id->id; ++it ) { + if (done(*it)) { + break; + } + auto itr = kv_index.find(boost::make_tuple(table_id->id, it->primary_key)); + add_val(*itr); + } + } + else { + using namespace eosio::chain; + EOS_ASSERT(db_backing_store == backing_store_type::ROCKSDB, + contract_table_query_exception, + "Support for configured backing_store has not been added to get_table_by_scope"); + const auto& kv_database = db.kv_db(); + using key_type = backing_store::db_key_value_format::key_type; + + // derive the "root" of the float64 secondary key space to determine where it ends (upper), then may need to recalculate + auto lower_key = backing_store::db_key_value_format::create_full_prefix_key(code, scope, sec_producers_table, key_type::sec_double); + auto upper_key = lower_key.next(); + if (lower.to_uint64_t() == 0) { + lower_key = backing_store::db_key_value_format::create_full_prefix_secondary_key(code, scope, sec_producers_table, float64_t{lower.to_uint64_t()}); + } - const auto& kv_index = d.get_index(); - const auto& secondary_index = d.get_index().indices(); - const auto& secondary_index_by_primary = secondary_index.get(); - const auto& secondary_index_by_secondary = secondary_index.get(); + auto session = kv_database.get_kv_undo_stack()->top(); + auto retrieve_primary_key = [&code,&scope,&producers_table,&session](uint64_t primary_key) { + auto full_primary_key = backing_store::db_key_value_format::create_full_primary_key(code, scope, producers_table, primary_key); + auto value = session.read(full_primary_key); + return *value; + }; - read_only::get_producers_result result; - const auto stopTime = fc::time_point::now() + fc::microseconds(1000 * 10); // 10ms - vector data; + using done_func = decltype(done); + using add_val_func = decltype(add_val); + using retrieve_prim_func = decltype(retrieve_primary_key); + + struct f64_secondary_key_receiver + : backing_store::single_type_error_receiver, + contract_table_query_exception> { + f64_secondary_key_receiver(read_only::get_producers_result& result, done_func&& done, + add_val_func&& add_val, retrieve_prim_func&& retrieve_prim) + : result_(result), done_(done), add_val_(add_val), retrieve_primary_key_(retrieve_prim) {} + + void add_only_row(const backing_store::secondary_index_view& row) { + // needs to allow a second pass after limit is reached or time has passed, to allow "more" processing + if (done_(row)) { + finished_ = true; + } + else { + auto value = retrieve_primary_key_(row.primary_key); + add_val_(backing_store::primary_index_view::create(row.primary_key, value.data(), value.size())); + } + } - auto it = [&]{ - if(lower.to_uint64_t() == 0) - return secondary_index_by_secondary.lower_bound( - boost::make_tuple(secondary_table_id->id, to_softfloat64(std::numeric_limits::lowest()), 0)); - else - return secondary_index.project( - secondary_index_by_primary.lower_bound( - boost::make_tuple(secondary_table_id->id, lower.to_uint64_t()))); - }(); - - for( ; it != secondary_index_by_secondary.end() && it->t_id == secondary_table_id->id; ++it ) { - if (result.rows.size() >= p.limit || fc::time_point::now() > stopTime) { - result.more = name{it->primary_key}.to_string(); - break; - } - copy_inline_row(*kv_index.find(boost::make_tuple(table_id->id, it->primary_key)), data); - if (p.json) - result.rows.emplace_back( abis.binary_to_variant( abis.get_table_type(N(producers)), data, abi_serializer::create_yield_function( abi_serializer_max_time ), shorten_abi_errors ) ); - else - result.rows.emplace_back(fc::variant(data)); - } + void add_table_row(const backing_store::table_id_object_view& ) { + // used for only one table, so we already know the context of the table + } - result.total_producer_vote_weight = get_global_row(d, abi, abis, abi_serializer_max_time, shorten_abi_errors)["total_producer_vote_weight"].as_double(); + auto keep_processing_entries() { + return [&finished=finished_]() { + return !finished; + }; + }; + + read_only::get_producers_result& result_; + done_func done_; + add_val_func add_val_; + retrieve_prim_func retrieve_primary_key_; + bool finished_ = false; + }; + f64_secondary_key_receiver receiver(result, std::move(done), std::move(add_val), std::move(retrieve_primary_key)); + auto kpe = receiver.keep_processing_entries(); + backing_store::rocksdb_contract_db_table_writer> + writer(receiver, backing_store::key_context::standalone, kpe); + eosio::chain::backing_store::walk_rocksdb_entries_with_prefix(kv_database.get_kv_undo_stack(), lower_key, upper_key, writer); + }; + + constexpr name global = "global"_n; + const auto global_table_type = get_table_type(abi, global); + EOS_ASSERT(global_table_type == read_only::KEYi64, chain::contract_table_query_exception, "Invalid table type ${type} for table global", ("type",global_table_type)); + auto var = get_primary_key(config::system_account_name, config::system_account_name, global, global.to_uint64_t(), row_requirements::required, row_requirements::required, abis.get_table_type(global)); + result.total_producer_vote_weight = var["total_producer_vote_weight"].as_double(); return result; } catch (...) { read_only::get_producers_result result; @@ -1993,8 +2781,8 @@ read_only::get_producers_result read_only::get_producers( const read_only::get_p ("total_votes", 0.0f); // detect a legacy key and maintain API compatibility for those entries - if (p.authority.contains()) { - const auto& auth = p.authority.get(); + if (std::holds_alternative(p.authority)) { + const auto& auth = std::get(p.authority); if (auth.keys.size() == 1 && auth.keys.back().weight == auth.threshold) { row("producer_key", auth.keys.back().key); } @@ -2020,7 +2808,7 @@ read_only::get_producer_schedule_result read_only::get_producer_schedule( const template struct resolver_factory { static auto make(const Api* api, abi_serializer::yield_function_t yield) { - return [api, yield{std::move(yield)}](const account_name &name) -> optional { + return [api, yield{std::move(yield)}](const account_name &name) -> std::optional { const auto* accnt = api->db.db().template find(name); if (accnt != nullptr) { abi_def abi; @@ -2029,7 +2817,7 @@ struct resolver_factory { } } - return optional(); + return std::optional(); }; } }; @@ -2115,7 +2903,7 @@ read_only::get_scheduled_transactions( const read_only::get_scheduled_transactio fc::variant read_only::get_block(const read_only::get_block_params& params) const { signed_block_ptr block; - optional block_num; + std::optional block_num; EOS_ASSERT( !params.block_num_or_id.empty() && params.block_num_or_id.size() <= 64, chain::block_id_type_exception, @@ -2126,7 +2914,7 @@ fc::variant read_only::get_block(const read_only::get_block_params& params) cons block_num = fc::to_uint64(params.block_num_or_id); } catch( ... ) {} - if( block_num.valid() ) { + if( block_num ) { block = db.fetch_block_by_number( *block_num ); } else { try { @@ -2136,27 +2924,58 @@ fc::variant read_only::get_block(const read_only::get_block_params& params) cons EOS_ASSERT( block, unknown_block_exception, "Could not find block: ${block}", ("block", params.block_num_or_id)); + // serializes signed_block to variant in signed_block_v0 format fc::variant pretty_output; abi_serializer::to_variant(*block, pretty_output, make_resolver(this, abi_serializer::create_yield_function( abi_serializer_max_time )), abi_serializer::create_yield_function( abi_serializer_max_time )); - uint32_t ref_block_prefix = block->id()._hash[1]; + const auto id = block->calculate_id(); + const uint32_t ref_block_prefix = id._hash[1]; return fc::mutable_variant_object(pretty_output.get_object()) - ("id", block->id()) + ("id", id) ("block_num",block->block_num()) ("ref_block_prefix", ref_block_prefix); } +fc::variant read_only::get_block_info(const read_only::get_block_info_params& params) const { + + signed_block_ptr block; + try { + block = db.fetch_block_by_number( params.block_num ); + } catch (...) { + // assert below will handle the invalid block num + } + + EOS_ASSERT( block, unknown_block_exception, "Could not find block: ${block}", ("block", params.block_num)); + + const auto id = block->calculate_id(); + const uint32_t ref_block_prefix = id._hash[1]; + + return fc::mutable_variant_object () + ("block_num", block->block_num()) + ("ref_block_num", static_cast(block->block_num())) + ("id", id) + ("timestamp", block->timestamp) + ("producer", block->producer) + ("confirmed", block->confirmed) + ("previous", block->previous) + ("transaction_mroot", block->transaction_mroot) + ("action_mroot", block->action_mroot) + ("schedule_version", block->schedule_version) + ("producer_signature", block->producer_signature) + ("ref_block_prefix", ref_block_prefix); +} + fc::variant read_only::get_block_header_state(const get_block_header_state_params& params) const { block_state_ptr b; - optional block_num; + std::optional block_num; std::exception_ptr e; try { block_num = fc::to_uint64(params.block_num_or_id); } catch( ... ) {} - if( block_num.valid() ) { + if( block_num ) { b = db.fetch_block_state_by_number(*block_num); } else { try { @@ -2173,7 +2992,7 @@ fc::variant read_only::get_block_header_state(const get_block_header_state_param void read_write::push_block(read_write::push_block_params&& params, next_function next) { try { - app().get_method()(std::make_shared(std::move(params)), {}); + app().get_method()(std::make_shared( std::move( params ), true), {}); next(read_write::push_block_results{}); } catch ( boost::interprocess::bad_alloc& ) { chain_plugin::handle_db_exhaustion(); @@ -2184,18 +3003,42 @@ void read_write::push_block(read_write::push_block_params&& params, next_functio void read_write::push_transaction(const read_write::push_transaction_params& params, next_function next) { try { - auto pretty_input = std::make_shared(); + packed_transaction_v0 input_trx_v0; auto resolver = make_resolver(this, abi_serializer::create_yield_function( abi_serializer_max_time )); + packed_transaction_ptr input_trx; try { - abi_serializer::from_variant(params, *pretty_input, std::move( resolver ), abi_serializer::create_yield_function( abi_serializer_max_time )); + abi_serializer::from_variant(params, input_trx_v0, std::move( resolver ), abi_serializer::create_yield_function( abi_serializer_max_time )); + input_trx = std::make_shared( std::move( input_trx_v0 ), true ); } EOS_RETHROW_EXCEPTIONS(chain::packed_transaction_type_exception, "Invalid packed transaction") - app().get_method()(pretty_input, true, - [this, next](const fc::static_variant& result) -> void { - if (result.contains()) { - next(result.get()); + auto trx_trace = fc_create_trace_with_id("Transaction", input_trx->id()); + auto trx_span = fc_create_span(trx_trace, "HTTP Received"); + fc_add_tag(trx_span, "trx_id", input_trx->id()); + fc_add_tag(trx_span, "method", "push_transaction"); + + app().get_method()(input_trx, true, + [this, token=trx_trace.get_token(), input_trx, next] + (const std::variant& result) -> void { + + auto trx_span = fc_create_span_from_token(token, "Processed"); + fc_add_tag(trx_span, "trx_id", input_trx->id()); + + if (std::holds_alternative(result)) { + auto& eptr = std::get(result); + fc_add_tag(trx_span, "error", eptr->to_string()); + next(eptr); } else { - auto trx_trace_ptr = result.get(); + auto& trx_trace_ptr = std::get(result); + + fc_add_tag(trx_span, "block_num", trx_trace_ptr->block_num); + fc_add_tag(trx_span, "block_time", trx_trace_ptr->block_time.to_time_point()); + fc_add_tag(trx_span, "elapsed", trx_trace_ptr->elapsed.count()); + if( trx_trace_ptr->receipt ) { + fc_add_tag(trx_span, "status", std::string(trx_trace_ptr->receipt->status)); + } + if( trx_trace_ptr->except ) { + fc_add_tag(trx_span, "error", trx_trace_ptr->except->to_string()); + } try { fc::variant output; @@ -2265,12 +3108,12 @@ void read_write::push_transaction(const read_write::push_transaction_params& par } static void push_recurse(read_write* rw, int index, const std::shared_ptr& params, const std::shared_ptr& results, const next_function& next) { - auto wrapped_next = [=](const fc::static_variant& result) { - if (result.contains()) { - const auto& e = result.get(); + auto wrapped_next = [=](const std::variant& result) { + if (std::holds_alternative(result)) { + const auto& e = std::get(result); results->emplace_back( read_write::push_transaction_results{ transaction_id_type(), fc::mutable_variant_object( "error", e->to_detail_string() ) } ); } else { - const auto& r = result.get(); + const auto& r = std::get(result); results->emplace_back( r ); } @@ -2303,18 +3146,41 @@ void read_write::push_transactions(const read_write::push_transactions_params& p void read_write::send_transaction(const read_write::send_transaction_params& params, next_function next) { try { - auto pretty_input = std::make_shared(); + packed_transaction_v0 input_trx_v0; auto resolver = make_resolver(this, abi_serializer::create_yield_function( abi_serializer_max_time )); + packed_transaction_ptr input_trx; try { - abi_serializer::from_variant(params, *pretty_input, resolver, abi_serializer::create_yield_function( abi_serializer_max_time )); + abi_serializer::from_variant(params, input_trx_v0, std::move( resolver ), abi_serializer::create_yield_function( abi_serializer_max_time )); + input_trx = std::make_shared( std::move( input_trx_v0 ), true ); } EOS_RETHROW_EXCEPTIONS(chain::packed_transaction_type_exception, "Invalid packed transaction") - app().get_method()(pretty_input, true, - [this, next](const fc::static_variant& result) -> void { - if (result.contains()) { - next(result.get()); + auto trx_trace = fc_create_trace_with_id("Transaction", input_trx->id()); + auto trx_span = fc_create_span(trx_trace, "HTTP Received"); + fc_add_tag(trx_span, "trx_id", input_trx->id()); + fc_add_tag(trx_span, "method", "send_transaction"); + + app().get_method()(input_trx, true, + [this, token=trx_trace.get_token(), input_trx, next] + (const std::variant& result) -> void { + auto trx_span = fc_create_span_from_token(token, "Processed"); + fc_add_tag(trx_span, "trx_id", input_trx->id()); + + if (std::holds_alternative(result)) { + auto& eptr = std::get(result); + fc_add_tag(trx_span, "error", eptr->to_string()); + next(eptr); } else { - auto trx_trace_ptr = result.get(); + auto& trx_trace_ptr = std::get(result); + + fc_add_tag(trx_span, "block_num", trx_trace_ptr->block_num); + fc_add_tag(trx_span, "block_time", trx_trace_ptr->block_time.to_time_point()); + fc_add_tag(trx_span, "elapsed", trx_trace_ptr->elapsed.count()); + if( trx_trace_ptr->receipt ) { + fc_add_tag(trx_span, "status", std::string(trx_trace_ptr->receipt->status)); + } + if( trx_trace_ptr->except ) { + fc_add_tag(trx_span, "error", trx_trace_ptr->except->to_string()); + } try { fc::variant output; @@ -2437,16 +3303,17 @@ read_only::get_account_results read_only::get_account( const get_account_params& result.created = accnt_obj.creation_date; uint32_t greylist_limit = db.is_resource_greylisted(result.account_name) ? 1 : config::maximum_elastic_resource_multiplier; - result.net_limit = rm.get_account_net_limit_ex( result.account_name, greylist_limit).first; - result.cpu_limit = rm.get_account_cpu_limit_ex( result.account_name, greylist_limit).first; + const block_timestamp_type current_usage_time (db.head_block_time()); + result.net_limit.set( rm.get_account_net_limit_ex( result.account_name, greylist_limit, current_usage_time).first ); + if ( result.net_limit.last_usage_update_time && (result.net_limit.last_usage_update_time->slot == 0) ) { // account has no action yet + result.net_limit.last_usage_update_time = accnt_obj.creation_date; + } + result.cpu_limit.set( rm.get_account_cpu_limit_ex( result.account_name, greylist_limit, current_usage_time).first ); + if ( result.cpu_limit.last_usage_update_time && (result.cpu_limit.last_usage_update_time->slot == 0) ) { // account has no action yet + result.cpu_limit.last_usage_update_time = accnt_obj.creation_date; + } result.ram_usage = rm.get_account_ram_usage( result.account_name ); - if ( producer_plug ) { // producer_plug is null when called from chain_plugin_tests.cpp and get_table_tests.cpp - account_resource_limit subjective_cpu_bill_limit; - subjective_cpu_bill_limit.used = producer_plug->get_subjective_bill( result.account_name, fc::time_point::now() ); - result.subjective_cpu_bill_limit = subjective_cpu_bill_limit; - } - const auto& permissions = d.get_index(); auto perm = permissions.lower_bound( boost::make_tuple( params.account_name ) ); while( perm != permissions.end() && perm->owner == params.account_name ) { @@ -2472,88 +3339,40 @@ read_only::get_account_results read_only::get_account( const get_account_params& if( abi_serializer::to_abi(code_account.abi, abi) ) { abi_serializer abis( abi, abi_serializer::create_yield_function( abi_serializer_max_time ) ); - const auto token_code = N(eosio.token); + const auto token_code = "eosio.token"_n; auto core_symbol = extract_core_symbol(); - if (params.expected_core_symbol.valid()) + if (params.expected_core_symbol) core_symbol = *(params.expected_core_symbol); - const auto* t_id = d.find(boost::make_tuple( token_code, params.account_name, N(accounts) )); - if( t_id != nullptr ) { - const auto &idx = d.get_index(); - auto it = idx.find(boost::make_tuple( t_id->id, core_symbol.to_symbol_code() )); - if( it != idx.end() && it->value.size() >= sizeof(asset) ) { - asset bal; - fc::datastream ds(it->value.data(), it->value.size()); - fc::raw::unpack(ds, bal); - - if( bal.get_symbol().valid() && bal.get_symbol() == core_symbol ) { - result.core_liquid_balance = bal; - } + get_primary_key(token_code, params.account_name, "accounts"_n, core_symbol.to_symbol_code(), + row_requirements::optional, row_requirements::optional, [&core_symbol,&result](const asset& bal) { + if( bal.get_symbol().valid() && bal.get_symbol() == core_symbol ) { + result.core_liquid_balance = bal; } - } + }); - t_id = d.find(boost::make_tuple( config::system_account_name, params.account_name, N(userres) )); - if (t_id != nullptr) { - const auto &idx = d.get_index(); - auto it = idx.find(boost::make_tuple( t_id->id, params.account_name.to_uint64_t() )); - if ( it != idx.end() ) { - vector data; - copy_inline_row(*it, data); - result.total_resources = abis.binary_to_variant( "user_resources", data, abi_serializer::create_yield_function( abi_serializer_max_time ), shorten_abi_errors ); - } - } + result.total_resources = get_primary_key(config::system_account_name, params.account_name, "userres"_n, params.account_name.to_uint64_t(), + row_requirements::optional, row_requirements::optional, "user_resources", abis); - t_id = d.find(boost::make_tuple( config::system_account_name, params.account_name, N(delband) )); - if (t_id != nullptr) { - const auto &idx = d.get_index(); - auto it = idx.find(boost::make_tuple( t_id->id, params.account_name.to_uint64_t() )); - if ( it != idx.end() ) { - vector data; - copy_inline_row(*it, data); - result.self_delegated_bandwidth = abis.binary_to_variant( "delegated_bandwidth", data, abi_serializer::create_yield_function( abi_serializer_max_time ), shorten_abi_errors ); - } - } + result.self_delegated_bandwidth = get_primary_key(config::system_account_name, params.account_name, "delband"_n, params.account_name.to_uint64_t(), + row_requirements::optional, row_requirements::optional, "delegated_bandwidth", abis); - t_id = d.find(boost::make_tuple( config::system_account_name, params.account_name, N(refunds) )); - if (t_id != nullptr) { - const auto &idx = d.get_index(); - auto it = idx.find(boost::make_tuple( t_id->id, params.account_name.to_uint64_t() )); - if ( it != idx.end() ) { - vector data; - copy_inline_row(*it, data); - result.refund_request = abis.binary_to_variant( "refund_request", data, abi_serializer::create_yield_function( abi_serializer_max_time ), shorten_abi_errors ); - } - } + result.refund_request = get_primary_key(config::system_account_name, params.account_name, "refunds"_n, params.account_name.to_uint64_t(), + row_requirements::optional, row_requirements::optional, "refund_request", abis); - t_id = d.find(boost::make_tuple( config::system_account_name, config::system_account_name, N(voters) )); - if (t_id != nullptr) { - const auto &idx = d.get_index(); - auto it = idx.find(boost::make_tuple( t_id->id, params.account_name.to_uint64_t() )); - if ( it != idx.end() ) { - vector data; - copy_inline_row(*it, data); - result.voter_info = abis.binary_to_variant( "voter_info", data, abi_serializer::create_yield_function( abi_serializer_max_time ), shorten_abi_errors ); - } - } + result.voter_info = get_primary_key(config::system_account_name, config::system_account_name, "voters"_n, params.account_name.to_uint64_t(), + row_requirements::optional, row_requirements::optional, "voter_info", abis); - t_id = d.find(boost::make_tuple( config::system_account_name, config::system_account_name, N(rexbal) )); - if (t_id != nullptr) { - const auto &idx = d.get_index(); - auto it = idx.find(boost::make_tuple( t_id->id, params.account_name.to_uint64_t() )); - if( it != idx.end() ) { - vector data; - copy_inline_row(*it, data); - result.rex_info = abis.binary_to_variant( "rex_balance", data, abi_serializer::create_yield_function( abi_serializer_max_time ), shorten_abi_errors ); - } - } + result.rex_info = get_primary_key(config::system_account_name, config::system_account_name, "rexbal"_n, params.account_name.to_uint64_t(), + row_requirements::optional, row_requirements::optional, "rex_balance", abis); } return result; } -static variant action_abi_to_variant( const abi_def& abi, type_name action_type ) { - variant v; +static fc::variant action_abi_to_variant( const abi_def& abi, type_name action_type ) { + fc::variant v; auto it = std::find_if(abi.structs.begin(), abi.structs.end(), [&](auto& x){return x.name == action_type;}); if( it != abi.structs.end() ) to_variant( it->fields, v ); @@ -2614,10 +3433,10 @@ read_only::get_transaction_id_result read_only::get_transaction_id( const read_o account_query_db::get_accounts_by_authorizers_result read_only::get_accounts_by_authorizers( const account_query_db::get_accounts_by_authorizers_params& args) const { - EOS_ASSERT(aqdb.valid(), plugin_config_exception, "Account Queries being accessed when not enabled"); + EOS_ASSERT(aqdb.has_value(), plugin_config_exception, "Account Queries being accessed when not enabled"); return aqdb->get_accounts_by_authorizers(args); -} - +} + namespace detail { struct ram_market_exchange_state_t { asset ignore1; @@ -2632,31 +3451,37 @@ chain::symbol read_only::extract_core_symbol()const { symbol core_symbol(0); // The following code makes assumptions about the contract deployed on eosio account (i.e. the system contract) and how it stores its data. - const auto& d = db.db(); - const auto* t_id = d.find(boost::make_tuple( N(eosio), N(eosio), N(rammarket) )); - if( t_id != nullptr ) { - const auto &idx = d.get_index(); - auto it = idx.find(boost::make_tuple( t_id->id, eosio::chain::string_to_symbol_c(4,"RAMCORE") )); - if( it != idx.end() ) { - detail::ram_market_exchange_state_t ram_market_exchange_state; - - fc::datastream ds( it->value.data(), it->value.size() ); - - try { - fc::raw::unpack(ds, ram_market_exchange_state); - } catch( ... ) { - return core_symbol; - } - + get_primary_key("eosio"_n, "eosio"_n, "rammarket"_n, eosio::chain::string_to_symbol_c(4,"RAMCORE"), + row_requirements::optional, row_requirements::optional, [&core_symbol](const detail::ram_market_exchange_state_t& ram_market_exchange_state) { if( ram_market_exchange_state.core_symbol.get_symbol().valid() ) { core_symbol = ram_market_exchange_state.core_symbol.get_symbol(); } - } - } + }); return core_symbol; } +fc::variant read_only::get_primary_key(name code, name scope, name table, uint64_t primary_key, row_requirements require_table, + row_requirements require_primary, const std::string_view& type, bool as_json) const { + const abi_def abi = eosio::chain_apis::get_abi(db, code); + abi_serializer abis; + abis.set_abi(abi, abi_serializer::create_yield_function(abi_serializer_max_time)); + return get_primary_key(code, scope, table, primary_key, require_table, require_primary, type, abis, as_json); +} + +fc::variant read_only::get_primary_key(name code, name scope, name table, uint64_t primary_key, row_requirements require_table, + row_requirements require_primary, const std::string_view& type, const abi_serializer& abis, + bool as_json) const { + fc::variant val; + const auto valid = get_primary_key_internal(code, scope, table, primary_key, require_table, require_primary, get_primary_key_value(val, type, abis, as_json)); + return val; +} + +eosio::chain::backing_store_type read_only::get_backing_store() const { + const auto& kv_database = db.kv_db(); + return kv_database.get_backing_store(); +} + } // namespace chain_apis } // namespace eosio diff --git a/plugins/chain_plugin/include/eosio/chain_plugin/account_query_db.hpp b/plugins/chain_plugin/include/eosio/chain_plugin/account_query_db.hpp index 3fc7e5df0cf..ce957843197 100644 --- a/plugins/chain_plugin/include/eosio/chain_plugin/account_query_db.hpp +++ b/plugins/chain_plugin/include/eosio/chain_plugin/account_query_db.hpp @@ -64,8 +64,8 @@ namespace eosio::chain_apis { struct account_result { chain::name account_name; chain::name permission_name; - fc::optional authorizing_account; - fc::optional authorizing_key; + std::optional authorizing_account; + std::optional authorizing_key; chain::weight_type weight; uint32_t threshold; }; diff --git a/plugins/chain_plugin/include/eosio/chain_plugin/blockvault_sync_strategy.hpp b/plugins/chain_plugin/include/eosio/chain_plugin/blockvault_sync_strategy.hpp new file mode 100644 index 00000000000..b5268063665 --- /dev/null +++ b/plugins/chain_plugin/include/eosio/chain_plugin/blockvault_sync_strategy.hpp @@ -0,0 +1,119 @@ +#pragma once +#include +#include + +namespace eosio { +namespace blockvault { + +template +struct blockvault_sync_strategy : public sync_callback { + blockvault_sync_strategy(block_vault_interface* blockvault, BP& blockchain_provider, std::function shutdown, + std::function check_shutdown) + : _blockvault(blockvault) + , _blockchain_provider(blockchain_provider) + , _shutdown(shutdown) + , _check_shutdown(check_shutdown) + , _startup_run(false) + , _received_snapshot(false) { + EOS_ASSERT(nullptr != blockvault, plugin_exception, "block_vault_interface cannot be null"); + } + + ~blockvault_sync_strategy() { + if (_num_unlinkable_blocks) + wlog("${num} out of ${total} blocks received are unlinkable", + ("num", _num_unlinkable_blocks)("total", _num_blocks_received)); + } + + void run_startup() { + _blockchain_provider.do_non_snapshot_startup(_shutdown, _check_shutdown); + _startup_run = true; + } + + void do_sync() { + auto head_block = _blockchain_provider.chain->last_irreversible_block(); + if (nullptr != head_block) { + block_id_type bid = head_block->calculate_id(); + ilog("Requesting blockvault sync from block id ${id}, block_num ${num}", + ("id", (const string)bid)("num", head_block->block_num())); + _blockvault->sync(&bid, *this); + } else { + ilog("Requesting complete blockvault sync."); + _blockvault->sync(nullptr, *this); + } + + if (!_startup_run) { + ilog("Received no data from blockvault."); + run_startup(); + } + + ilog("Sync from blockvault completed. ${snap}. ${blks} blocks received. ${ulnk} blocks unlinkable", + ("snap", _received_snapshot ? "Got snapshot" : "No snapshot") + ("blks", _num_blocks_received)("ulnk", _num_unlinkable_blocks)); + } + + void on_snapshot(const char* snapshot_filename) override final { + ilog("Received snapshot from blockvault ${fn}", ("fn", snapshot_filename)); + EOS_ASSERT(!_received_snapshot, plugin_exception, "Received multiple snapshots from blockvault." ); + _received_snapshot = true; + + if (_check_shutdown()) { + _shutdown(); + } + + auto infile = std::ifstream(snapshot_filename, (std::ios::in | std::ios::binary)); + auto reader = std::make_shared(infile); + + _blockchain_provider.chain->startup(_shutdown, _check_shutdown, reader); + _startup_run = true; + + infile.close(); + + _snapshot_height = _blockchain_provider.chain->head_block_num(); + } + + void on_block(eosio::chain::signed_block_ptr block) override final { + if (0 == (_num_blocks_received % 100)) { + dlog("Received block number ${bn}", ("bn", block->block_num())); + } + + if (_check_shutdown()) { + _shutdown(); + } + + if (!_startup_run) { + run_startup(); + } + + try { + + ++_num_blocks_received; + auto rc = _blockchain_provider.incoming_blockvault_sync_method(block, + !(_received_snapshot && block->block_num() == _snapshot_height +1)); + + EOS_ASSERT(rc, plugin_exception, + "Unable to sync block from blockvault, block num=${bnum}, block id=${bid}", + ("bnum", block->block_num())("bid", block->calculate_id())); + } catch (unlinkable_block_exception& e) { + if (block->block_num() == 2) { + elog("Received unlinkable block 2. Please double check if --genesis-json and --genesis-timestamp are " + "correctly specified"); + throw e; + } + ++_num_unlinkable_blocks; + } + } + + private: + block_vault_interface* _blockvault; + BP& _blockchain_provider; + std::function _shutdown; + std::function _check_shutdown; + bool _startup_run; + bool _received_snapshot; + uint32_t _num_unlinkable_blocks = 0; + uint32_t _num_blocks_received = 0; + uint32_t _snapshot_height = 0; +}; + +} // namespace blockvault +} // namespace eosio diff --git a/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp index db80a8262b9..44e35d36694 100644 --- a/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp @@ -11,7 +11,13 @@ #include #include #include +#include #include +#include +#include +#include +#include +#include #include #include @@ -19,6 +25,7 @@ #include #include +#include namespace fc { class variant; } @@ -32,7 +39,6 @@ namespace eosio { using chain::public_key_type; using chain::transaction; using chain::transaction_id_type; - using fc::optional; using boost::container::flat_set; using chain::asset; using chain::symbol; @@ -42,8 +48,6 @@ namespace eosio { using chain::abi_def; using chain::abi_serializer; -class producer_plugin; - namespace chain_apis { struct empty{}; @@ -64,6 +68,8 @@ Type convert_to_type(const string& str, const string& desc) { } FC_RETHROW_EXCEPTIONS(warn, "Could not convert ${desc} string '${str}' to key type.", ("desc", desc)("str",str) ) } +uint64_t convert_to_type(const eosio::name &n, const string &desc); + template<> uint64_t convert_to_type(const string& str, const string& desc); @@ -80,20 +86,30 @@ template<> string convert_to_string(const float128_t& source, const string& key_type, const string& encode_type, const string& desc); +class keep_processing { +public: + explicit keep_processing(fc::microseconds&& duration = fc::milliseconds(10)) : end_time_(fc::time_point::now() + duration) {} + + fc::microseconds time_remaining() const { return end_time_ - fc::time_point::now(); } + bool operator()() const { + return time_remaining().count() >= 0; + } +private: + fc::time_point end_time_; +}; + class read_only { const controller& db; - const fc::optional& aqdb; + const std::optional& aqdb; const fc::microseconds abi_serializer_max_time; bool shorten_abi_errors = true; - const producer_plugin* producer_plug; public: static const string KEYi64; - read_only(const controller& db, const fc::optional& aqdb, const fc::microseconds& abi_serializer_max_time, const producer_plugin* producer_plug) - : db(db), aqdb(aqdb), abi_serializer_max_time(abi_serializer_max_time), producer_plug(producer_plug) { - } - + read_only(const controller& db, const std::optional& aqdb, const fc::microseconds& abi_serializer_max_time) + : db(db), aqdb(aqdb), abi_serializer_max_time(abi_serializer_max_time) {} + void validate() const {} void set_shorten_abi_errors( bool f ) { shorten_abi_errors = f; } @@ -101,40 +117,41 @@ class read_only { using get_info_params = empty; struct get_info_results { - string server_version; - chain::chain_id_type chain_id; - uint32_t head_block_num = 0; - uint32_t last_irreversible_block_num = 0; - chain::block_id_type last_irreversible_block_id; - chain::block_id_type head_block_id; - fc::time_point head_block_time; - account_name head_block_producer; - - uint64_t virtual_block_cpu_limit = 0; - uint64_t virtual_block_net_limit = 0; - - uint64_t block_cpu_limit = 0; - uint64_t block_net_limit = 0; - //string recent_slots; - //double participation_rate = 0; - optional server_version_string; - optional fork_db_head_block_num; - optional fork_db_head_block_id; - optional server_full_version_string; + string server_version; + chain::chain_id_type chain_id; + uint32_t head_block_num = 0; + uint32_t last_irreversible_block_num = 0; + chain::block_id_type last_irreversible_block_id; + chain::block_id_type head_block_id; + fc::time_point head_block_time; + account_name head_block_producer; + + uint64_t virtual_block_cpu_limit = 0; + uint64_t virtual_block_net_limit = 0; + + uint64_t block_cpu_limit = 0; + uint64_t block_net_limit = 0; + //string recent_slots; + //double participation_rate = 0; + std::optional server_version_string; + std::optional fork_db_head_block_num; + std::optional fork_db_head_block_id; + std::optional server_full_version_string; + std::optional last_irreversible_block_time; }; get_info_results get_info(const get_info_params&) const; struct get_activated_protocol_features_params { - optional lower_bound; - optional upper_bound; - uint32_t limit = 10; - bool search_by_block_num = false; - bool reverse = false; + std::optional lower_bound; + std::optional upper_bound; + uint32_t limit = 10; + bool search_by_block_num = false; + bool reverse = false; }; struct get_activated_protocol_features_results { - fc::variants activated_protocol_features; - optional more; + fc::variants activated_protocol_features; + std::optional more; }; get_activated_protocol_features_results get_activated_protocol_features( const get_activated_protocol_features_params& params )const; @@ -143,7 +160,22 @@ class read_only { name producer_name; }; - using account_resource_limit = chain::resource_limits::account_resource_limit; + // account_resource_info holds similar data members as in account_resource_limit, but decoupling making them independently to be refactored in future + struct account_resource_info { + int64_t used = 0; + int64_t available = 0; + int64_t max = 0; + std::optional last_usage_update_time; // optional for backward nodeos support + std::optional current_used; // optional for backward nodeos support + void set( const chain::resource_limits::account_resource_limit& arl) + { + used = arl.used; + available = arl.available; + max = arl.max; + last_usage_update_time = arl.last_usage_update_time; + current_used = arl.current_used; + } + }; struct get_account_results { name account_name; @@ -154,14 +186,14 @@ class read_only { fc::time_point last_code_update; fc::time_point created; - optional core_liquid_balance; + std::optional core_liquid_balance; int64_t ram_quota = 0; int64_t net_weight = 0; int64_t cpu_weight = 0; - account_resource_limit net_limit; - account_resource_limit cpu_limit; + account_resource_info net_limit; + account_resource_info cpu_limit; int64_t ram_usage = 0; vector permissions; @@ -171,13 +203,11 @@ class read_only { fc::variant refund_request; fc::variant voter_info; fc::variant rex_info; - - optional subjective_cpu_bill_limit; }; struct get_account_params { - name account_name; - optional expected_core_symbol; + name account_name; + std::optional expected_core_symbol; }; get_account_results get_account( const get_account_params& params )const; @@ -187,12 +217,12 @@ class read_only { string wast; string wasm; fc::sha256 code_hash; - optional abi; + std::optional abi; }; struct get_code_params { name account_name; - bool code_as_wasm = false; + bool code_as_wasm = true; }; struct get_code_hash_results { @@ -206,7 +236,7 @@ class read_only { struct get_abi_results { name account_name; - optional abi; + std::optional abi; }; struct get_abi_params { @@ -224,15 +254,15 @@ class read_only { }; struct get_raw_abi_params { - name account_name; - optional abi_hash; + name account_name; + std::optional abi_hash; }; struct get_raw_abi_results { - name account_name; - fc::sha256 code_hash; - fc::sha256 abi_hash; - optional abi; + name account_name; + fc::sha256 code_hash; + fc::sha256 abi_hash; + std::optional abi; }; @@ -289,6 +319,12 @@ class read_only { fc::variant get_block(const get_block_params& params) const; + struct get_block_info_params { + uint32_t block_num; + }; + + fc::variant get_block_info(const get_block_info_params& params) const; + struct get_block_header_state_params { string block_num_or_id; }; @@ -296,36 +332,53 @@ class read_only { fc::variant get_block_header_state(const get_block_header_state_params& params) const; struct get_table_rows_params { - bool json = false; - name code; - string scope; - name table; - string table_key; - string lower_bound; - string upper_bound; - uint32_t limit = 10; - string key_type; // type of key specified by index_position - string index_position; // 1 - primary (first), 2 - secondary index (in order defined by multi_index), 3 - third index, etc - string encode_type{"dec"}; //dec, hex , default=dec - optional reverse; - optional show_payer; // show RAM pyer + bool json = false; + name code; + string scope; + name table; + string table_key; + string lower_bound; + string upper_bound; + uint32_t limit = 10; + string key_type; // type of key specified by index_position + string index_position; // 1 - primary (first), 2 - secondary index (in order defined by multi_index), 3 - third index, etc + string encode_type{"dec"}; //dec, hex , default=dec + std::optional reverse; + std::optional show_payer; // show RAM pyer }; + struct get_kv_table_rows_params { + bool json = false; // true if you want output rows in json format, false as variant + name code; // name of contract + name table; // name of kv table, + name index_name; // name of index index + string encode_type; // encoded type for values in index_value/lower_bound/upper_bound + string index_value; // index value for point query. If this is set, it is processed as a point query + string lower_bound; // lower bound value of index of index_name. If index_value is not set and lower_bound is not set, return from the beginning of range in the prefix + string upper_bound; // upper bound value of index of index_name, If index_value is not set and upper_bound is not set, It is set to the beginning of the next prefix range. + uint32_t limit = 10; // max number of rows + bool reverse = false; // if true output rows in reverse order + bool show_payer = false; + }; + struct get_table_rows_result { vector rows; ///< one row per item, either encoded as hex String or JSON object bool more = false; ///< true if last element in data is not the end and sizeof data() < limit string next_key; ///< fill lower_bound with this value to fetch more rows + string next_key_bytes; ///< fill lower_bound with this value to fetch more rows with encode-type of "bytes" }; get_table_rows_result get_table_rows( const get_table_rows_params& params )const; + get_table_rows_result get_kv_table_rows( const get_kv_table_rows_params& params )const; + struct get_table_by_scope_params { - name code; // mandatory - name table; // optional, act as filter - string lower_bound; // lower bound of scope, optional - string upper_bound; // upper bound of scope, optional - uint32_t limit = 10; - optional reverse; + name code; // mandatory + name table; // optional, act as filter + string lower_bound; // lower bound of scope, optional + string upper_bound; // upper bound of scope, optional + uint32_t limit = 10; + std::optional reverse; }; struct get_table_by_scope_result_row { name code; @@ -342,9 +395,9 @@ class read_only { get_table_by_scope_result get_table_by_scope( const get_table_by_scope_params& params )const; struct get_currency_balance_params { - name code; - name account; - optional symbol; + name code; + name account; + std::optional symbol; }; vector get_currency_balance( const get_currency_balance_params& params )const; @@ -401,32 +454,226 @@ class read_only { get_scheduled_transactions_result get_scheduled_transactions( const get_scheduled_transactions_params& params ) const; - static void copy_inline_row(const chain::key_value_object& obj, vector& data) { + eosio::chain::backing_store_type get_backing_store() const; + + enum class row_requirements { required, optional }; + template + bool get_primary_key_internal(name code, name scope, name table, uint64_t primary_key, row_requirements require_table, + row_requirements require_primary, Function&& f) const { + const auto db_backing_store = get_backing_store(); + if (db_backing_store == eosio::chain::backing_store_type::CHAINBASE) { + const auto* const table_id = + db.db().find(boost::make_tuple(code, scope, table)); + if (require_table == row_requirements::optional && !table_id) { + return false; + } + EOS_ASSERT(table_id, chain::contract_table_query_exception, + "Missing code: ${code}, scope: ${scope}, table: ${table}", + ("code",code.to_string())("scope",scope.to_string())("table",table.to_string())); + const auto& kv_index = db.db().get_index(); + const auto it = kv_index.find(boost::make_tuple(table_id->id, primary_key)); + if (require_primary == row_requirements::optional && it == kv_index.end()) { + return false; + } + EOS_ASSERT(it != kv_index.end(), chain::contract_table_query_exception, + "Missing row for primary_key: ${primary} in code: ${code}, scope: ${scope}, table: ${table}", + ("primary", primary_key)("code",code.to_string())("scope",scope.to_string()) + ("table",table.to_string())); + f(*it); + return true; + } + else { + using namespace eosio::chain; + EOS_ASSERT(db_backing_store == backing_store_type::ROCKSDB, + chain::contract_table_query_exception, + "Support for configured backing_store has not been added to get_primary_key"); + const auto& kv_database = db.kv_db(); + const auto full_key = chain::backing_store::db_key_value_format::create_full_primary_key(code, scope, table, primary_key); + auto current_session = kv_database.get_kv_undo_stack()->top(); + const auto value = current_session.read(full_key); + // check if we didn't actually find the key + if (!value) { + // only need to bother to do table search if we require it, so that we can report the correct error + if (require_table == row_requirements::required) { + const auto whole_table_prefix(backing_store::db_key_value_format::create_full_key_prefix(full_key, backing_store::db_key_value_format::end_of_prefix::pre_type)); + const auto value = current_session.read(whole_table_prefix); + EOS_ASSERT(value, chain::contract_table_query_exception, + "Missing code: ${code}, scope: ${scope}, table: ${table}", + ("code",code.to_string())("scope",scope.to_string())("table",table.to_string())); + } + EOS_ASSERT(require_primary == row_requirements::optional, chain::contract_table_query_exception, + "Missing row for primary_key: ${primary} in code: ${code}, scope: ${scope}, table: ${table}", + ("primary", primary_key)("code",code.to_string())("scope",scope.to_string()) + ("table",table.to_string())); + return false; + } + f(chain::backing_store::primary_index_view::create(primary_key, value->data(), value->size())); + return true; + } + } + + template + bool get_primary_key(name code, name scope, name table, uint64_t primary_key, row_requirements require_table, + row_requirements require_primary, Function&& f) const { + auto ret = get_primary_key_internal(code, scope, table, primary_key, require_table, require_primary, [&f](const auto& obj) { + if( obj.value.size() >= sizeof(T) ) { + T t; + fc::datastream ds(obj.value.data(), obj.value.size()); + fc::raw::unpack(ds, t); + + f(t); + } + }); + return ret; + } + + fc::variant get_primary_key(name code, name scope, name table, uint64_t primary_key, row_requirements require_table, + row_requirements require_primary, const std::string_view& type, bool as_json = true) const; + fc::variant get_primary_key(name code, name scope, name table, uint64_t primary_key, row_requirements require_table, + row_requirements require_primary, const std::string_view& type, const abi_serializer& abis, + bool as_json = true) const; + + auto get_primary_key_value(const std::string_view& type, const abi_serializer& abis, bool as_json = true) const { + return [table_type=std::string{type},abis,as_json,this](fc::variant& result_var, const auto& obj) { + vector data; + read_only::copy_inline_row(obj, data); + if (as_json) { + result_var = abis.binary_to_variant(table_type, data, abi_serializer::create_yield_function( abi_serializer_max_time ), shorten_abi_errors ); + } + else { + result_var = fc::variant(data); + } + }; + } + + auto get_primary_key_value(fc::variant& result_var, const std::string_view& type, const abi_serializer& abis, bool as_json = true) const { + auto get_primary = get_primary_key_value(type, abis, as_json); + return [&result_var,get_primary{std::move(get_primary)}](const auto& obj) { + return get_primary(result_var, obj); + }; + } + + auto get_primary_key_value(name table, const abi_serializer& abis, bool as_json, const std::optional& show_payer) const { + return [abis,table,show_payer,as_json,this](const auto& obj) -> fc::variant { + fc::variant data_var; + auto get_prim = get_primary_key_value(data_var, abis.get_table_type(table), abis, as_json); + get_prim(obj); + + if( show_payer && *show_payer ) { + return fc::mutable_variant_object("data", std::move(data_var))("payer", obj.payer); + } else { + return data_var; + } + }; + } + + template + static void copy_inline_row(const KeyValueObj& obj, vector& data) { data.resize( obj.value.size() ); memcpy( data.data(), obj.value.data(), obj.value.size() ); } + template + struct primary_key_receiver + : chain::backing_store::single_type_error_receiver, chain::backing_store::primary_index_view, chain::contract_table_query_exception> { + primary_key_receiver(Func f) : f_(f) {} + + void add_only_row(const chain::backing_store::primary_index_view& row) { + if(!f_(row)) + done_ = true; + } + + void add_table_row(const chain::backing_store::table_id_object_view& ) { + // used for only one table, so we already know the context of the table + } + + auto keep_processing_entries() { + return [&done=done_]() { + return !done; + }; + }; + + Func f_; + bool done_ = false; + }; + template void walk_key_value_table(const name& code, const name& scope, const name& table, Function f) const { - const auto& d = db.db(); - const auto* t_id = d.find(boost::make_tuple(code, scope, table)); - if (t_id != nullptr) { - const auto &idx = d.get_index(); - decltype(t_id->id) next_tid(t_id->id._id + 1); - auto lower = idx.lower_bound(boost::make_tuple(t_id->id)); - auto upper = idx.lower_bound(boost::make_tuple(next_tid)); - - for (auto itr = lower; itr != upper; ++itr) { - if (!f(*itr)) { - break; + const auto db_backing_store = get_backing_store(); + if (db_backing_store == eosio::chain::backing_store_type::CHAINBASE) { + const auto& d = db.db(); + const auto* t_id = d.find(boost::make_tuple(code, scope, table)); + if (t_id != nullptr) { + const auto &idx = d.get_index(); + decltype(t_id->id) next_tid(t_id->id._id + 1); + auto lower = idx.lower_bound(boost::make_tuple(t_id->id)); + auto upper = idx.lower_bound(boost::make_tuple(next_tid)); + + for (auto itr = lower; itr != upper; ++itr) { + if (!f(*itr)) { + break; + } } } } + else { + using namespace eosio::chain; + EOS_ASSERT(db_backing_store == backing_store_type::ROCKSDB, + chain::contract_table_query_exception, + "Support for configured backing_store has not been added to get_primary_key"); + primary_key_receiver receiver(f); + auto kp = receiver.keep_processing_entries(); + backing_store::rocksdb_contract_db_table_writer, std::decay_t < decltype(kp)>> writer(receiver, backing_store::key_context::standalone, kp); + const auto& kv_database = db.kv_db(); + using key_type = chain::backing_store::db_key_value_format::key_type; + auto start = chain::backing_store::db_key_value_format::create_full_prefix_key(code, scope, table, key_type::primary); + auto end = start.next(); + eosio::chain::backing_store::walk_rocksdb_entries_with_prefix(kv_database.get_kv_undo_stack(), start, end, writer); + } } static uint64_t get_table_index_name(const read_only::get_table_rows_params& p, bool& primary); + + template + struct secondary_key_receiver + : chain::backing_store::single_type_error_receiver, chain::backing_store::secondary_index_view, chain::contract_table_query_exception> { + secondary_key_receiver(read_only::get_table_rows_result& result, Function f, const read_only::get_table_rows_params& params) + : result_(result), f_(f), params_(params) {} + + void add_only_row(const chain::backing_store::secondary_index_view& row) { + // needs to allow a second pass after limit is reached or time has passed, to allow "more" processing + if (reached_limit_ || !kp_()) { + result_.more = true; + result_.next_key = convert_to_string(row.secondary_key, params_.key_type, params_.encode_type, "next_key - next lower bound"); + done_ = true; + } + else { + f_(row, result_.rows); + reached_limit_ |= result_.rows.size() >= params_.limit; + } + } + + void add_table_row(const chain::backing_store::table_id_object_view& ) { + // used for only one table, so we already know the context of the table + } + + auto keep_processing_entries() { + return [&done=done_]() { + return !done; + }; + }; + + read_only::get_table_rows_result& result_; + Function f_; + const read_only::get_table_rows_params& params_; + bool reached_limit_ = false; + bool done_ = false; + keep_processing kp_; + }; + + template read_only::get_table_rows_result get_table_rows_by_seckey( const read_only::get_table_rows_params& p, const abi_def& abi, ConvFn conv )const { read_only::get_table_rows_result result; @@ -438,83 +685,114 @@ class read_only { abis.set_abi(abi, abi_serializer::create_yield_function( abi_serializer_max_time ) ); bool primary = false; const uint64_t table_with_index = get_table_index_name(p, primary); - const auto* t_id = d.find(boost::make_tuple(p.code, scope, p.table)); - const auto* index_t_id = d.find(boost::make_tuple(p.code, scope, name(table_with_index))); - if( t_id != nullptr && index_t_id != nullptr ) { - using secondary_key_type = std::result_of_t; - static_assert( std::is_same::value, "Return type of conv does not match type of secondary key for IndexType" ); - - const auto& secidx = d.get_index(); - auto lower_bound_lookup_tuple = std::make_tuple( index_t_id->id._id, - eosio::chain::secondary_key_traits::true_lowest(), - std::numeric_limits::lowest() ); - auto upper_bound_lookup_tuple = std::make_tuple( index_t_id->id._id, - eosio::chain::secondary_key_traits::true_highest(), - std::numeric_limits::max() ); - - if( p.lower_bound.size() ) { - if( p.key_type == "name" ) { - name s(p.lower_bound); - SecKeyType lv = convert_to_type( s.to_string(), "lower_bound name" ); // avoids compiler error - std::get<1>(lower_bound_lookup_tuple) = conv( lv ); + using secondary_key_type = std::result_of_t; + static_assert( std::is_same::value, "Return type of conv does not match type of secondary key for IndexType" ); + auto secondary_key_lower = eosio::chain::secondary_key_traits::true_lowest(); + const auto primary_key_lower = std::numeric_limits::lowest(); + auto secondary_key_upper = eosio::chain::secondary_key_traits::true_highest(); + const auto primary_key_upper = std::numeric_limits::max(); + if( p.lower_bound.size() ) { + if( p.key_type == "name" ) { + if constexpr (std::is_same_v) { + SecKeyType lv = convert_to_type(name{p.lower_bound}, "lower_bound name"); + secondary_key_lower = conv( lv ); } else { - SecKeyType lv = convert_to_type( p.lower_bound, "lower_bound" ); - std::get<1>(lower_bound_lookup_tuple) = conv( lv ); + EOS_ASSERT(false, chain::contract_table_query_exception, "Invalid key type of eosio::name ${nm} for lower bound", ("nm", p.lower_bound)); } + } else { + SecKeyType lv = convert_to_type( p.lower_bound, "lower_bound" ); + secondary_key_lower = conv( lv ); } + } - if( p.upper_bound.size() ) { - if( p.key_type == "name" ) { - name s(p.upper_bound); - SecKeyType uv = convert_to_type( s.to_string(), "upper_bound name" ); - std::get<1>(upper_bound_lookup_tuple) = conv( uv ); + if( p.upper_bound.size() ) { + if( p.key_type == "name" ) { + if constexpr (std::is_same_v) { + SecKeyType uv = convert_to_type(name{p.upper_bound}, "upper_bound name"); + secondary_key_upper = conv( uv ); } else { - SecKeyType uv = convert_to_type( p.upper_bound, "upper_bound" ); - std::get<1>(upper_bound_lookup_tuple) = conv( uv ); + EOS_ASSERT(false, chain::contract_table_query_exception, "Invalid key type of eosio::name ${nm} for upper bound", ("nm", p.upper_bound)); } + } else { + SecKeyType uv = convert_to_type( p.upper_bound, "upper_bound" ); + secondary_key_upper = conv( uv ); } - - if( upper_bound_lookup_tuple < lower_bound_lookup_tuple ) - return result; - - auto walk_table_row_range = [&]( auto itr, auto end_itr ) { - auto cur_time = fc::time_point::now(); - auto end_time = cur_time + fc::microseconds(1000 * 10); /// 10ms max time - vector data; - for( unsigned int count = 0; cur_time <= end_time && count < p.limit && itr != end_itr; ++itr, cur_time = fc::time_point::now() ) { - const auto* itr2 = d.find( boost::make_tuple(t_id->id, itr->primary_key) ); - if( itr2 == nullptr ) continue; - copy_inline_row(*itr2, data); - - fc::variant data_var; - if( p.json ) { - data_var = abis.binary_to_variant( abis.get_table_type(p.table), data, abi_serializer::create_yield_function( abi_serializer_max_time ), shorten_abi_errors ); - } else { - data_var = fc::variant( data ); + } + if( secondary_key_upper < secondary_key_lower ) + return result; + + const bool reverse = p.reverse && *p.reverse; + const auto db_backing_store = get_backing_store(); + auto get_prim_key_val = get_primary_key_value(p.table, abis, p.json, p.show_payer); + if (db_backing_store == eosio::chain::backing_store_type::CHAINBASE) { + const auto* t_id = d.find(boost::make_tuple(p.code, scope, p.table)); + const auto* index_t_id = d.find(boost::make_tuple(p.code, scope, name(table_with_index))); + if( t_id != nullptr && index_t_id != nullptr ) { + + const auto& secidx = d.get_index(); + auto lower_bound_lookup_tuple = std::make_tuple( index_t_id->id._id, + secondary_key_lower, + primary_key_lower ); + auto upper_bound_lookup_tuple = std::make_tuple( index_t_id->id._id, + secondary_key_upper, + primary_key_upper ); + + auto walk_table_row_range = [&]( auto itr, auto end_itr ) { + keep_processing kp; + vector data; + for( unsigned int count = 0; kp() && count < p.limit && itr != end_itr; ++itr ) { + const auto* itr2 = d.find( boost::make_tuple(t_id->id, itr->primary_key) ); + if( itr2 == nullptr ) continue; + + result.rows.emplace_back( get_prim_key_val(*itr2) ); + + ++count; } - - if( p.show_payer && *p.show_payer ) { - result.rows.emplace_back( fc::mutable_variant_object("data", std::move(data_var))("payer", itr->payer) ); - } else { - result.rows.emplace_back( std::move(data_var) ); + if( itr != end_itr ) { + result.more = true; + result.next_key = convert_to_string(itr->secondary_key, p.key_type, p.encode_type, "next_key - next lower bound"); } + }; - ++count; - } - if( itr != end_itr ) { - result.more = true; - result.next_key = convert_to_string(itr->secondary_key, p.key_type, p.encode_type, "next_key - next lower bound"); + auto lower = secidx.lower_bound( lower_bound_lookup_tuple ); + auto upper = secidx.upper_bound( upper_bound_lookup_tuple ); + if( reverse ) { + walk_table_row_range( boost::make_reverse_iterator(upper), boost::make_reverse_iterator(lower) ); + } else { + walk_table_row_range( lower, upper ); } - }; - - auto lower = secidx.lower_bound( lower_bound_lookup_tuple ); - auto upper = secidx.upper_bound( upper_bound_lookup_tuple ); - if( p.reverse && *p.reverse ) { - walk_table_row_range( boost::make_reverse_iterator(upper), boost::make_reverse_iterator(lower) ); - } else { - walk_table_row_range( lower, upper ); } } + else { + using namespace eosio::chain; + EOS_ASSERT(db_backing_store == backing_store_type::ROCKSDB, + chain::contract_table_query_exception, + "Support for configured backing_store has not been added to get_primary_key"); + const auto context = (reverse) ? backing_store::key_context::standalone_reverse : backing_store::key_context::standalone; + auto lower = chain::backing_store::db_key_value_format::create_full_prefix_secondary_key(p.code, scope, name(table_with_index), secondary_key_lower); + auto upper = chain::backing_store::db_key_value_format::create_full_prefix_secondary_key(p.code, scope, name(table_with_index), secondary_key_upper); + if (reverse) { + lower = eosio::session::shared_bytes::truncate_key(lower); + } + // since upper is either the upper_bound of a forward search, or the reverse iterator <= for the beginning of the end of + // this secondary type, we need to move it to just before the beginning of the next type + upper = upper.next(); + const auto& kv_database = db.kv_db(); + auto session = kv_database.get_kv_undo_stack()->top(); + auto get_primary = [code=p.code,scope,table=p.table,&session,&get_prim_key_val](const chain::backing_store::secondary_index_view& row, vector& rows) { + auto full_key = chain::backing_store::db_key_value_format::create_full_primary_key(code, scope, table, row.primary_key); + auto value = session.read(full_key); + if( !value ) return; + + rows.emplace_back(get_prim_key_val(chain::backing_store::primary_index_view::create(row.primary_key, value->data(), value->size()))); + }; + using secondary_receiver = secondary_key_receiver; + secondary_receiver receiver(result, get_primary, p); + auto kp = receiver.keep_processing_entries(); + backing_store::rocksdb_contract_db_table_writer> writer(receiver, context, kp); + eosio::chain::backing_store::walk_rocksdb_entries_with_prefix(kv_database.get_kv_undo_stack(), lower, upper, writer); + } + return result; } @@ -523,72 +801,105 @@ class read_only { read_only::get_table_rows_result result; const auto& d = db.db(); - uint64_t scope = convert_to_type(p.scope, "scope"); + name scope { convert_to_type(p.scope, "scope") }; abi_serializer abis; abis.set_abi(abi, abi_serializer::create_yield_function( abi_serializer_max_time )); - const auto* t_id = d.find(boost::make_tuple(p.code, name(scope), p.table)); - if( t_id != nullptr ) { - const auto& idx = d.get_index(); - auto lower_bound_lookup_tuple = std::make_tuple( t_id->id, std::numeric_limits::lowest() ); - auto upper_bound_lookup_tuple = std::make_tuple( t_id->id, std::numeric_limits::max() ); - - if( p.lower_bound.size() ) { - if( p.key_type == "name" ) { - name s(p.lower_bound); - std::get<1>(lower_bound_lookup_tuple) = s.to_uint64_t(); - } else { - auto lv = convert_to_type( p.lower_bound, "lower_bound" ); - std::get<1>(lower_bound_lookup_tuple) = lv; - } + + auto primary_lower = std::numeric_limits::lowest(); + auto primary_upper = std::numeric_limits::max(); + + if( p.lower_bound.size() ) { + if( p.key_type == "name" ) { + name s(p.lower_bound); + primary_lower = s.to_uint64_t(); + } else { + auto lv = convert_to_type( p.lower_bound, "lower_bound" ); + primary_lower = lv; } + } - if( p.upper_bound.size() ) { - if( p.key_type == "name" ) { - name s(p.upper_bound); - std::get<1>(upper_bound_lookup_tuple) = s.to_uint64_t(); - } else { - auto uv = convert_to_type( p.upper_bound, "upper_bound" ); - std::get<1>(upper_bound_lookup_tuple) = uv; - } + if( p.upper_bound.size() ) { + if( p.key_type == "name" ) { + name s(p.upper_bound); + primary_upper = s.to_uint64_t(); + } else { + auto uv = convert_to_type( p.upper_bound, "upper_bound" ); + primary_upper = uv; } + } - if( upper_bound_lookup_tuple < lower_bound_lookup_tuple ) - return result; - - auto walk_table_row_range = [&]( auto itr, auto end_itr ) { - auto cur_time = fc::time_point::now(); - auto end_time = cur_time + fc::microseconds(1000 * 10); /// 10ms max time - vector data; - for( unsigned int count = 0; cur_time <= end_time && count < p.limit && itr != end_itr; ++count, ++itr, cur_time = fc::time_point::now() ) { - copy_inline_row(*itr, data); - - fc::variant data_var; - if( p.json ) { - data_var = abis.binary_to_variant( abis.get_table_type(p.table), data, abi_serializer::create_yield_function( abi_serializer_max_time ), shorten_abi_errors ); - } else { - data_var = fc::variant( data ); + if( primary_upper < primary_lower ) + return result; + + auto get_prim_key = get_primary_key_value(p.table, abis, p.json, p.show_payer); + auto handle_more = [&result,&p](const auto& row) { + result.more = true; + result.next_key = convert_to_string(row.primary_key, p.key_type, p.encode_type, "next_key - next lower bound"); + }; + + const bool reverse = p.reverse && *p.reverse; + const auto db_backing_store = get_backing_store(); + if (db_backing_store == eosio::chain::backing_store_type::CHAINBASE) { + const auto* t_id = d.find(boost::make_tuple(p.code, scope, p.table)); + if( t_id != nullptr ) { + const auto& idx = d.get_index(); + auto lower_bound_lookup_tuple = std::make_tuple( t_id->id, primary_lower ); + auto upper_bound_lookup_tuple = std::make_tuple( t_id->id, primary_upper ); + + auto walk_table_row_range = [&]( auto itr, auto end_itr ) { + keep_processing kp; + vector data; + for( unsigned int count = 0; kp() && count < p.limit && itr != end_itr; ++count, ++itr ) { + result.rows.emplace_back( get_prim_key(*itr) ); } - - if( p.show_payer && *p.show_payer ) { - result.rows.emplace_back( fc::mutable_variant_object("data", std::move(data_var))("payer", itr->payer) ); - } else { - result.rows.emplace_back( std::move(data_var) ); + if( itr != end_itr ) { + handle_more(*itr); } + }; + + auto lower = idx.lower_bound( lower_bound_lookup_tuple ); + auto upper = idx.upper_bound( upper_bound_lookup_tuple ); + if( reverse ) { + walk_table_row_range( boost::make_reverse_iterator(upper), boost::make_reverse_iterator(lower) ); + } else { + walk_table_row_range( lower, upper ); + } + } + } + else { + using namespace eosio::chain; + EOS_ASSERT(db_backing_store == backing_store_type::ROCKSDB, + chain::contract_table_query_exception, + "Support for configured backing_store has not been added to get_primary_key"); + const auto context = (reverse) ? backing_store::key_context::standalone_reverse : backing_store::key_context::standalone; + auto lower = chain::backing_store::db_key_value_format::create_full_primary_key(p.code, scope, p.table, primary_lower); + auto upper = chain::backing_store::db_key_value_format::create_full_primary_key(p.code, scope, p.table, primary_upper); + if (reverse) { + lower = eosio::session::shared_bytes::truncate_key(lower); + } + // since upper is either the upper_bound of a forward search, or the reverse iterator <= for the beginning of the end of + // this secondary type, we need to move it to just before the beginning of the next type + upper = upper.next(); + const auto& kv_database = db.kv_db(); + + keep_processing kp; + auto filter_primary_key = [&kp,&result,&p,&get_prim_key,&handle_more](const backing_store::primary_index_view& row) { + if (!kp() || result.rows.size() >= p.limit) { + handle_more(row); + return false; } - if( itr != end_itr ) { - result.more = true; - result.next_key = convert_to_string(itr->primary_key, p.key_type, p.encode_type, "next_key - next lower bound"); + else { + result.rows.emplace_back(get_prim_key(row)); + return true; } }; - auto lower = idx.lower_bound( lower_bound_lookup_tuple ); - auto upper = idx.upper_bound( upper_bound_lookup_tuple ); - if( p.reverse && *p.reverse ) { - walk_table_row_range( boost::make_reverse_iterator(upper), boost::make_reverse_iterator(lower) ); - } else { - walk_table_row_range( lower, upper ); - } + using primary_receiver = primary_key_receiver; + primary_receiver receiver(filter_primary_key); + auto keep_processing_entries = receiver.keep_processing_entries(); + backing_store::rocksdb_contract_db_table_writer> writer(receiver, context, keep_processing_entries); + eosio::chain::backing_store::walk_rocksdb_entries_with_prefix(kv_database.get_kv_undo_stack(), lower, upper, writer); } return result; } @@ -610,7 +921,7 @@ class read_write { read_write(controller& db, const fc::microseconds& abi_serializer_max_time, bool api_accept_transactions); void validate() const; - using push_block_params = chain::signed_block; + using push_block_params = chain::signed_block_v0; using push_block_results = empty; void push_block(push_block_params&& params, chain::plugin_interface::next_function next); @@ -706,7 +1017,7 @@ class read_write { class chain_plugin : public plugin { public: - APPBASE_PLUGIN_REQUIRES() + APPBASE_PLUGIN_REQUIRES((blockvault_client_plugin)) chain_plugin(); virtual ~chain_plugin(); @@ -716,18 +1027,17 @@ class chain_plugin : public plugin { void plugin_initialize(const variables_map& options); void plugin_startup(); void plugin_shutdown(); + void handle_sighup() override; chain_apis::read_write get_read_write_api() { return chain_apis::read_write(chain(), get_abi_serializer_max_time(), api_accept_transactions()); } chain_apis::read_only get_read_only_api() const; - + bool accept_block( const chain::signed_block_ptr& block, const chain::block_id_type& id ); void accept_transaction(const chain::packed_transaction_ptr& trx, chain::plugin_interface::next_function next); - bool block_is_on_preferred_chain(const chain::block_id_type& block_id); - static bool recover_reversible_blocks( const fc::path& db_dir, uint32_t cache_size, - optional new_db_dir = optional(), + std::optional new_db_dir = std::optional(), uint32_t truncate_at_block = 0 ); @@ -757,7 +1067,7 @@ class chain_plugin : public plugin { static void handle_db_exhaustion(); static void handle_bad_alloc(); - + bool account_queries_enabled() const; private: static void log_guard_exception(const chain::guard_exception& e); @@ -773,16 +1083,19 @@ FC_REFLECT(eosio::chain_apis::read_only::get_info_results, (server_version)(chain_id)(head_block_num)(last_irreversible_block_num)(last_irreversible_block_id) (head_block_id)(head_block_time)(head_block_producer) (virtual_block_cpu_limit)(virtual_block_net_limit)(block_cpu_limit)(block_net_limit) - (server_version_string)(fork_db_head_block_num)(fork_db_head_block_id)(server_full_version_string) ) + (server_version_string)(fork_db_head_block_num)(fork_db_head_block_id)(server_full_version_string) + (last_irreversible_block_time) ) FC_REFLECT(eosio::chain_apis::read_only::get_activated_protocol_features_params, (lower_bound)(upper_bound)(limit)(search_by_block_num)(reverse) ) FC_REFLECT(eosio::chain_apis::read_only::get_activated_protocol_features_results, (activated_protocol_features)(more) ) FC_REFLECT(eosio::chain_apis::read_only::get_block_params, (block_num_or_id)) +FC_REFLECT(eosio::chain_apis::read_only::get_block_info_params, (block_num)) FC_REFLECT(eosio::chain_apis::read_only::get_block_header_state_params, (block_num_or_id)) FC_REFLECT( eosio::chain_apis::read_write::push_transaction_results, (transaction_id)(processed) ) FC_REFLECT( eosio::chain_apis::read_only::get_table_rows_params, (json)(code)(scope)(table)(table_key)(lower_bound)(upper_bound)(limit)(key_type)(index_position)(encode_type)(reverse)(show_payer) ) -FC_REFLECT( eosio::chain_apis::read_only::get_table_rows_result, (rows)(more)(next_key) ); +FC_REFLECT( eosio::chain_apis::read_only::get_kv_table_rows_params, (json)(code)(table)(index_name)(encode_type)(index_value)(lower_bound)(upper_bound)(limit)(reverse)(show_payer) ) +FC_REFLECT( eosio::chain_apis::read_only::get_table_rows_result, (rows)(more)(next_key)(next_key_bytes) ); FC_REFLECT( eosio::chain_apis::read_only::get_table_by_scope_params, (code)(table)(lower_bound)(upper_bound)(limit)(reverse) ) FC_REFLECT( eosio::chain_apis::read_only::get_table_by_scope_result_row, (code)(scope)(table)(payer)(count)); @@ -801,10 +1114,11 @@ FC_REFLECT( eosio::chain_apis::read_only::get_producer_schedule_result, (active) FC_REFLECT( eosio::chain_apis::read_only::get_scheduled_transactions_params, (json)(lower_bound)(limit) ) FC_REFLECT( eosio::chain_apis::read_only::get_scheduled_transactions_result, (transactions)(more) ); +FC_REFLECT( eosio::chain_apis::read_only::account_resource_info, (used)(available)(max)(last_usage_update_time)(current_used) ) FC_REFLECT( eosio::chain_apis::read_only::get_account_results, (account_name)(head_block_num)(head_block_time)(privileged)(last_code_update)(created) (core_liquid_balance)(ram_quota)(net_weight)(cpu_weight)(net_limit)(cpu_limit)(ram_usage)(permissions) - (total_resources)(self_delegated_bandwidth)(refund_request)(voter_info)(rex_info)(subjective_cpu_bill_limit) ) + (total_resources)(self_delegated_bandwidth)(refund_request)(voter_info)(rex_info) ) // @swap code_hash FC_REFLECT( eosio::chain_apis::read_only::get_code_results, (account_name)(code_hash)(wast)(wasm)(abi) ) FC_REFLECT( eosio::chain_apis::read_only::get_code_hash_results, (account_name)(code_hash) ) @@ -824,4 +1138,3 @@ FC_REFLECT( eosio::chain_apis::read_only::abi_bin_to_json_params, (code)(action) FC_REFLECT( eosio::chain_apis::read_only::abi_bin_to_json_result, (args) ) FC_REFLECT( eosio::chain_apis::read_only::get_required_keys_params, (transaction)(available_keys) ) FC_REFLECT( eosio::chain_apis::read_only::get_required_keys_result, (required_keys) ) - diff --git a/plugins/chain_plugin/test/CMakeLists.txt b/plugins/chain_plugin/test/CMakeLists.txt index d937ee66d53..f152229dc75 100644 --- a/plugins/chain_plugin/test/CMakeLists.txt +++ b/plugins/chain_plugin/test/CMakeLists.txt @@ -1,5 +1,11 @@ add_executable( test_account_query_db test_account_query_db.cpp ) +add_executable( test_blockvault_sync_strategy test_blockvault_sync_strategy.cpp ) +add_executable( test_chain_plugin test_chain_plugin.cpp ) target_link_libraries( test_account_query_db chain_plugin eosio_testing) +target_link_libraries( test_blockvault_sync_strategy chain_plugin eosio_testing) +target_link_libraries( test_chain_plugin chain_plugin eosio_testing) -add_test(NAME test_account_query_db COMMAND plugins/chain_plugin/test/test_account_query_db WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) \ No newline at end of file +add_test(NAME test_account_query_db COMMAND plugins/chain_plugin/test/test_account_query_db WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME test_blockvault_sync_strategy COMMAND plugins/chain_plugin/test/test_blockvault_sync_strategy WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME test_chain_plugin COMMAND plugins/chain_plugin/test/test_chain_plugin WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/plugins/chain_plugin/test/test_account_query_db.cpp b/plugins/chain_plugin/test/test_account_query_db.cpp index 40989cee704..c8882d91a53 100644 --- a/plugins/chain_plugin/test/test_account_query_db.cpp +++ b/plugins/chain_plugin/test/test_account_query_db.cpp @@ -1,6 +1,6 @@ #define BOOST_TEST_MODULE account_query_db #include -#include +#include #include #include #include @@ -21,7 +21,7 @@ using params = account_query_db::get_accounts_by_authorizers_params; using results = account_query_db::get_accounts_by_authorizers_result; bool find_account_name(results rst, account_name name){ - for (const auto acc : rst.accounts){ + for (const auto& acc : rst.accounts){ if (acc.account_name == name){ return true; } @@ -29,7 +29,7 @@ bool find_account_name(results rst, account_name name){ return false; } bool find_account_auth(results rst, account_name name, permission_name perm){ - for (const auto acc : rst.accounts){ + for (const auto& acc : rst.accounts){ if (acc.account_name == name && acc.permission_name == perm) return true; } @@ -50,7 +50,7 @@ BOOST_FIXTURE_TEST_CASE(newaccount_test, TESTER) { try { produce_blocks(10); - account_name tester_account = N(tester); + account_name tester_account = "tester"_n; const auto trace_ptr = create_account(tester_account); aq_db.cache_transaction_trace(trace_ptr); produce_block(); @@ -65,24 +65,24 @@ BOOST_FIXTURE_TEST_CASE(newaccount_test, TESTER) { try { BOOST_FIXTURE_TEST_CASE(updateauth_test, TESTER) { try { - // instantiate an account_query_db - auto aq_db = account_query_db(*control); + // instantiate an account_query_db + auto aq_db = account_query_db(*control); - //link aq_db to the `accepted_block` signal on the controller - auto c = control->accepted_block.connect([&](const block_state_ptr& blk) { - aq_db.commit_block( blk); - }); + //link aq_db to the `accepted_block` signal on the controller + auto c = control->accepted_block.connect([&](const block_state_ptr& blk) { + aq_db.commit_block( blk); + }); - produce_blocks(10); + produce_blocks(10); - const auto& tester_account = N(tester); + const auto& tester_account = "tester"_n; const string role = "first"; produce_block(); create_account(tester_account); const auto trace_ptr = push_action(config::system_account_name, updateauth::get_name(), tester_account, fc::mutable_variant_object() ("account", tester_account) - ("permission", N(role)) + ("permission", "role"_n) ("parent", "active") ("auth", authority(get_public_key(tester_account, role), 5)) ); @@ -93,7 +93,7 @@ BOOST_FIXTURE_TEST_CASE(updateauth_test, TESTER) { try { pars.keys.emplace_back(get_public_key(tester_account, role)); const auto results = aq_db.get_accounts_by_authorizers(pars); - BOOST_TEST_REQUIRE(find_account_auth(results, tester_account, N(role)) == true); + BOOST_TEST_REQUIRE(find_account_auth(results, tester_account, "role"_n) == true); } FC_LOG_AND_RETHROW() } @@ -115,13 +115,13 @@ BOOST_AUTO_TEST_CASE(future_fork_test) { try { } // produce a block on node A with a new account and permission - const auto& tester_account = N(tester); + const auto& tester_account = "tester"_n; const string role = "first"; node_a.create_account(tester_account); const auto trace_ptr = node_a.push_action(config::system_account_name, updateauth::get_name(), tester_account, fc::mutable_variant_object() ("account", tester_account) - ("permission", N(role)) + ("permission", "role"_n) ("parent", "active") ("auth", authority(node_a.get_public_key(tester_account, role), 5)) ); @@ -132,7 +132,7 @@ BOOST_AUTO_TEST_CASE(future_fork_test) { try { pars.keys.emplace_back(node_a.get_public_key(tester_account, role)); const auto pre_results = aq_db.get_accounts_by_authorizers(pars); - BOOST_TEST_REQUIRE(find_account_auth(pre_results, tester_account, N(role)) == true); + BOOST_TEST_REQUIRE(find_account_auth(pre_results, tester_account, "role"_n) == true); // have node B take over from head-1 and produce "future" blocks to overtake node_a.push_block(node_b.produce_block(fc::milliseconds(config::block_interval_ms * 100))); @@ -162,22 +162,22 @@ BOOST_AUTO_TEST_CASE(fork_test) { try { } // produce a block on node A with a new account and permission - const auto& tester_account = N(tester); - const auto& tester_account2 = N(tester2); + const auto& tester_account = "tester"_n; + const auto& tester_account2 = "tester2"_n; const string role = "first"; node_a.create_account(tester_account); node_a.create_account(tester_account2); const auto trace_ptr = node_a.push_action(config::system_account_name, updateauth::get_name(), tester_account, fc::mutable_variant_object() ("account", tester_account) - ("permission", N(role)) + ("permission", "role"_n) ("parent", "active") ("auth", authority(node_a.get_public_key(tester_account, role), 5)), 1 ); aq_db.cache_transaction_trace(trace_ptr); const auto trace_ptr2 = node_a.push_action(config::system_account_name, updateauth::get_name(), tester_account2, fc::mutable_variant_object() ("account", tester_account2) - ("permission", N(role)) + ("permission", "role"_n) ("parent", "active") ("auth", authority(node_a.get_public_key(tester_account2, role), 5)), 2 ); @@ -188,7 +188,7 @@ BOOST_AUTO_TEST_CASE(fork_test) { try { pars.keys.emplace_back(node_a.get_public_key(tester_account, role)); const auto pre_results = aq_db.get_accounts_by_authorizers(pars); - BOOST_TEST_REQUIRE(find_account_auth(pre_results, tester_account, N(role)) == true); + BOOST_TEST_REQUIRE(find_account_auth(pre_results, tester_account, "role"_n) == true); // have node B take over from head-1 and also update permissions node_b.create_account(tester_account); @@ -196,14 +196,14 @@ BOOST_AUTO_TEST_CASE(fork_test) { try { const auto trace_ptr3 = node_b.push_action(config::system_account_name, updateauth::get_name(), tester_account, fc::mutable_variant_object() ("account", tester_account) - ("permission", N(role)) + ("permission", "role"_n) ("parent", "active") ("auth", authority(node_b.get_public_key(tester_account, role), 6)), 1 ); aq_db.cache_transaction_trace(trace_ptr3); const auto trace_ptr4 = node_b.push_action(config::system_account_name, updateauth::get_name(), tester_account2, fc::mutable_variant_object() ("account", tester_account2) - ("permission", N(role)) + ("permission", "role"_n) ("parent", "active") ("auth", authority(node_b.get_public_key(tester_account2, role), 6)), 2 ); @@ -214,14 +214,14 @@ BOOST_AUTO_TEST_CASE(fork_test) { try { const auto trace_ptr5 = node_b.push_action(config::system_account_name, updateauth::get_name(), tester_account, fc::mutable_variant_object() ("account", tester_account) - ("permission", N(role)) + ("permission", "role"_n) ("parent", "active") ("auth", authority(node_b.get_public_key(tester_account, role), 5)), 3 ); aq_db.cache_transaction_trace(trace_ptr5); const auto trace_ptr6 = node_b.push_action(config::system_account_name, updateauth::get_name(), tester_account2, fc::mutable_variant_object() ("account", tester_account2) - ("permission", N(role)) + ("permission", "role"_n) ("parent", "active") ("auth", authority(node_b.get_public_key(tester_account2, role), 5)), 4 ); @@ -237,5 +237,6 @@ BOOST_AUTO_TEST_CASE(fork_test) { try { } FC_LOG_AND_RETHROW() } + BOOST_AUTO_TEST_SUITE_END() diff --git a/plugins/chain_plugin/test/test_blockvault_sync_strategy.cpp b/plugins/chain_plugin/test/test_blockvault_sync_strategy.cpp new file mode 100644 index 00000000000..684b343b665 --- /dev/null +++ b/plugins/chain_plugin/test/test_blockvault_sync_strategy.cpp @@ -0,0 +1,155 @@ +#define BOOST_TEST_MODULE blockvault_sync_strategy +#include +#include +#include +#include +#include + +#ifdef NON_VALIDATING_TEST +#define TESTER tester +#else +#define TESTER validating_tester +#endif + +using namespace eosio; +using namespace eosio::chain; +using namespace eosio::testing; +using namespace eosio::chain_apis; +using namespace eosio::blockvault; + +struct mock_genesis_t {}; + +struct mock_signed_block_t { + block_id_type _id; + uint32_t _block_num; + block_id_type calculate_id() { return _id; } + uint32_t block_num() { return _block_num; } +}; + +struct mock_chain_t { + void startup(std::function shutdown, std::function check_shutdown, + std::shared_ptr reader) { + _shutdown = shutdown; + _check_shutdown = check_shutdown; + _reader = reader; + _startup_reader_called = true; + } + + mock_signed_block_t* _last_irreversible_block = nullptr; + mock_signed_block_t* last_irreversible_block() { return _last_irreversible_block; } + + std::function _shutdown; + std::function _check_shutdown; + std::shared_ptr _reader; + + bool _startup_reader_called; + + uint32_t _head_block_num = 0; + uint32_t head_block_num() {return _head_block_num;} +}; + +struct mock_blockvault_t : public block_vault_interface { + eosio::chain::block_id_type _previous_block_id{}; + bool _previous_block_id_sent = false; + + virtual void async_propose_constructed_block(uint32_t lib, + eosio::chain::signed_block_ptr block, + std::function handler) override {} + virtual void async_append_external_block(uint32_t lib, eosio::chain::signed_block_ptr block, + std::function handler) override {} + + virtual bool propose_snapshot(watermark_t watermark, const char* snapshot_filename) override { return true; } + virtual void sync(const eosio::chain::block_id_type* previous_block_id, sync_callback& callback) override { + if (nullptr != previous_block_id) { + _previous_block_id = *previous_block_id; + _previous_block_id_sent = true; + } + } +}; + +struct mock_chain_plugin_t { + mock_chain_plugin_t() { + _accept_block_rc = true; + chain = std::make_unique(); + } + + bool incoming_blockvault_sync_method(const chain::signed_block_ptr& block, bool check_connectivity) { + _block = block; + return _accept_block_rc; + } + + bool _accept_block_rc; + signed_block_ptr _block; + block_id_type _id; + std::unique_ptr chain; + + bool _startup_non_snapshot_called = false; + + void do_non_snapshot_startup(std::function shutdown, std::function check_shutdown) { + _startup_non_snapshot_called = true; + } +}; + +BOOST_AUTO_TEST_SUITE(blockvault_sync_strategy_tests) + +BOOST_FIXTURE_TEST_CASE(empty_previous_block_id_test, TESTER) { + try { + + mock_chain_plugin_t plugin; + mock_blockvault_t bv; + + auto shutdown = []() { return false; }; + auto check_shutdown = []() { return false; }; + + blockvault_sync_strategy uut(&bv, plugin, shutdown, check_shutdown); + uut.do_sync(); + + BOOST_TEST(!bv._previous_block_id_sent); + } + FC_LOG_AND_RETHROW() +} + +BOOST_FIXTURE_TEST_CASE(nonempty_previous_block_id_test, TESTER) { + try { + + mock_chain_plugin_t plugin; + mock_blockvault_t bv; + auto shutdown = []() { return false; }; + auto check_shutdown = []() { return false; }; + std::string bid_hex("deadbabe000000000000000000000000000000000000000000000000deadbeef"); + chain::block_id_type bid(bid_hex); + mock_signed_block_t lib; + lib._block_num = 100; + lib._id = bid; + + plugin.chain->_last_irreversible_block = &lib; + + blockvault_sync_strategy uut(&bv, plugin, shutdown, check_shutdown); + uut.do_sync(); + + BOOST_TEST(bv._previous_block_id == bid); + } + FC_LOG_AND_RETHROW() +} + +BOOST_FIXTURE_TEST_CASE(on_block_no_snapshot, TESTER) { + try { + + mock_chain_plugin_t plugin; + mock_blockvault_t bv; + plugin.chain = std::make_unique(); + + auto shutdown = []() { return false; }; + auto check_shutdown = []() { return false; }; + blockvault_sync_strategy uut(&bv, plugin, shutdown, check_shutdown); + auto b = produce_empty_block(); + + uut.on_block(b); + BOOST_TEST(plugin.chain->_reader == nullptr); + BOOST_TEST(plugin._startup_non_snapshot_called); + BOOST_TEST(!plugin.chain->_startup_reader_called); + BOOST_TEST(plugin._block->calculate_id() == b->calculate_id()); + } + FC_LOG_AND_RETHROW() +} +BOOST_AUTO_TEST_SUITE_END() diff --git a/plugins/chain_plugin/test/test_chain_plugin.cpp b/plugins/chain_plugin/test/test_chain_plugin.cpp new file mode 100644 index 00000000000..fbcb8570170 --- /dev/null +++ b/plugins/chain_plugin/test/test_chain_plugin.cpp @@ -0,0 +1,511 @@ +#define BOOST_TEST_MODULE chain_plugin +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef NON_VALIDATING_TEST +#define TESTER tester +#else +#define TESTER validating_tester +#endif + +using namespace eosio; +using namespace eosio::chain; +using namespace eosio::chain_apis; +using namespace eosio::testing; +using namespace eosio::chain_apis; +using namespace fc; + +using mvo = fc::mutable_variant_object; + +class chain_plugin_tester : public TESTER { +public: + + action_result push_action( const account_name& signer, const action_name &name, const variant_object &data, bool auth = true ) { + string action_type_name = abi_ser.get_action_type(name); + + action act; + act.account = config::system_account_name; + act.name = name; + act.data = abi_ser.variant_to_binary( action_type_name, data, abi_serializer::create_yield_function(abi_serializer_max_time) ); + + return base_tester::push_action( std::move(act), (auth ? signer : signer == "bob111111111"_n ? "alice1111111"_n : "bob111111111"_n).to_uint64_t() ); + } + + action_result deposit( const account_name& owner, const asset& amount ) { + return push_action( name(owner), "deposit"_n, mvo() + ("owner", owner) + ("amount", amount) + ); + } + + void transfer( name from, name to, const asset& amount, name manager = config::system_account_name ) { + base_tester::push_action( "eosio.token"_n, "transfer"_n, manager, mutable_variant_object() + ("from", from) + ("to", to ) + ("quantity", amount) + ("memo", "") + ); + } + + action_result stake( const account_name& from, const account_name& to, const asset& net, const asset& cpu ) { + return push_action( name(from), "delegatebw"_n, mvo() + ("from", from) + ("receiver", to) + ("stake_net_quantity", net) + ("stake_cpu_quantity", cpu) + ("transfer", 0 ) + ); + } + + action_result stake( const account_name& acnt, const asset& net, const asset& cpu ) { + return stake( acnt, acnt, net, cpu ); + } + + asset get_balance( const account_name& act ) { + vector data = get_row_by_account( "eosio.token"_n, act, "accounts"_n, name(symbol(CORE_SYMBOL).to_symbol_code().value) ); + return data.empty() ? asset(0, symbol(CORE_SYMBOL)) : token_abi_ser.binary_to_variant("account", data, abi_serializer::create_yield_function( abi_serializer_max_time ))["balance"].as(); + } + + transaction_trace_ptr create_account_with_resources( account_name a, account_name creator, uint32_t ram_bytes = 8000 ) { + signed_transaction trx; + set_transaction_headers(trx); + + authority owner_auth; + owner_auth = authority( get_public_key( a, "owner" ) ); + + trx.actions.emplace_back( vector{{creator,config::active_name}}, + newaccount{ + .creator = creator, + .name = a, + .owner = owner_auth, + .active = authority( get_public_key( a, "active" ) ) + }); + + trx.actions.emplace_back( get_action( config::system_account_name, "buyrambytes"_n, vector{{creator,config::active_name}}, + mvo() + ("payer", creator) + ("receiver", a) + ("bytes", ram_bytes) ) + ); + trx.actions.emplace_back( get_action( config::system_account_name, "delegatebw"_n, vector{{creator,config::active_name}}, + mvo() + ("from", creator) + ("receiver", a) + ("stake_net_quantity", core_from_string("10.0000") ) + ("stake_cpu_quantity", core_from_string("10.0000") ) + ("transfer", 0 ) + ) + ); + + set_transaction_headers(trx); + trx.sign( get_private_key( creator, "active" ), control->get_chain_id() ); + return push_transaction( trx ); + } + + transaction_trace_ptr create_account_with_resources( account_name a, account_name creator, asset ramfunds, bool multisig, + asset net = core_from_string("10.0000"), asset cpu = core_from_string("10.0000") ) { + signed_transaction trx; + set_transaction_headers(trx); + + authority owner_auth; + if (multisig) { + // multisig between account's owner key and creators active permission + owner_auth = authority(2, {key_weight{get_public_key( a, "owner" ), 1}}, {permission_level_weight{{creator, config::active_name}, 1}}); + } else { + owner_auth = authority( get_public_key( a, "owner" ) ); + } + + authority active_auth( get_public_key( a, "active" ) ); + + auto sort_permissions = []( authority& auth ) { + std::sort( auth.accounts.begin(), auth.accounts.end(), + []( const permission_level_weight& lhs, const permission_level_weight& rhs ) { + return lhs.permission < rhs.permission; + } + ); + }; + + { + FC_ASSERT( owner_auth.threshold <= std::numeric_limits::max(), "threshold is too high" ); + FC_ASSERT( active_auth.threshold <= std::numeric_limits::max(), "threshold is too high" ); + owner_auth.accounts.push_back( permission_level_weight{ {a, config::eosio_code_name}, + static_cast(owner_auth.threshold) } ); + sort_permissions(owner_auth); + active_auth.accounts.push_back( permission_level_weight{ {a, config::eosio_code_name}, + static_cast(active_auth.threshold) } ); + sort_permissions(active_auth); + } + + trx.actions.emplace_back( vector{{creator,config::active_name}}, + newaccount{ + .creator = creator, + .name = a, + .owner = owner_auth, + .active = authority( get_public_key( a, "active" ) ) + }); + + + trx.actions.emplace_back( get_action( config::system_account_name, "buyram"_n, vector{{creator,config::active_name}}, + mvo() + ("payer", creator) + ("receiver", a) + ("quant", ramfunds) ) + ); + + trx.actions.emplace_back( get_action( config::system_account_name, "delegatebw"_n, vector{{creator,config::active_name}}, + mvo() + ("from", creator) + ("receiver", a) + ("stake_net_quantity", net ) + ("stake_cpu_quantity", cpu ) + ("transfer", 0 ) + ) + ); + + set_transaction_headers(trx); + trx.sign( get_private_key( creator, "active" ), control->get_chain_id() ); + return push_transaction( trx ); + } + + void create_currency( name contract, name manager, asset maxsupply ) { + auto act = mutable_variant_object() + ("issuer", manager ) + ("maximum_supply", maxsupply ); + + base_tester::push_action(contract, "create"_n, contract, act ); + } + + void issue( name to, const asset& amount, name manager = config::system_account_name ) { + base_tester::push_action( "eosio.token"_n, "issue"_n, manager, mutable_variant_object() + ("to", to ) + ("quantity", amount ) + ("memo", "") + ); + } + void setup_system_accounts(){ + create_accounts({ "eosio.token"_n, "eosio.ram"_n, "eosio.ramfee"_n, "eosio.stake"_n, + "eosio.bpay"_n, "eosio.vpay"_n, "eosio.saving"_n, "eosio.names"_n, "eosio.rex"_n }); + + set_code( "eosio.token"_n, contracts::eosio_token_wasm() ); + set_abi( "eosio.token"_n, contracts::eosio_token_abi().data() ); + + { + const auto& accnt = control->db().get( "eosio.token"_n ); + abi_def abi; + BOOST_CHECK_EQUAL(abi_serializer::to_abi(accnt.abi, abi), true); + token_abi_ser.set_abi(abi, abi_serializer::create_yield_function( abi_serializer_max_time )); + } + + create_currency( "eosio.token"_n, config::system_account_name, core_from_string("10000000000.0000") ); + issue(config::system_account_name, core_from_string("1000000000.0000")); + BOOST_CHECK_EQUAL( core_from_string("1000000000.0000"), get_balance( name("eosio") ) ); + + set_code( config::system_account_name, contracts::eosio_system_wasm() ); + set_abi( config::system_account_name, contracts::eosio_system_abi().data() ); + + base_tester::push_action(config::system_account_name, "init"_n, + config::system_account_name, mutable_variant_object() + ("version", 0) + ("core", CORE_SYM_STR)); + + { + const auto& accnt = control->db().get( config::system_account_name ); + abi_def abi; + BOOST_CHECK_EQUAL(abi_serializer::to_abi(accnt.abi, abi), true); + abi_ser.set_abi(abi, abi_serializer::create_yield_function( abi_serializer_max_time )); + } + + } + + read_only::get_account_results get_account_info(const account_name acct){ + auto account_object = control->get_account(acct); + read_only::get_account_params params = { account_object.name }; + chain_apis::read_only plugin(*(control.get()), {}, fc::microseconds::maximum()); + return plugin.get_account(params); + } + + transaction_trace_ptr setup_producer_accounts( const std::vector& accounts ) { + account_name creator(config::system_account_name); + signed_transaction trx; + set_transaction_headers(trx); + asset cpu = core_from_string("80.0000"); + asset net = core_from_string("80.0000"); + asset ram = core_from_string("1.0000"); + + for (const auto& a: accounts) { + authority owner_auth( get_public_key( a, "owner" ) ); + trx.actions.emplace_back( vector{{creator,config::active_name}}, + newaccount{ + .creator = creator, + .name = a, + .owner = owner_auth, + .active = authority( get_public_key( a, "active" ) ) + }); + + trx.actions.emplace_back( get_action( config::system_account_name, "buyram"_n, vector{ {creator, config::active_name} }, + mvo() + ("payer", creator) + ("receiver", a) + ("quant", ram) ) + ); + + trx.actions.emplace_back( get_action( config::system_account_name, "delegatebw"_n, vector{ {creator, config::active_name} }, + mvo() + ("from", creator) + ("receiver", a) + ("stake_net_quantity", net) + ("stake_cpu_quantity", cpu ) + ("transfer", 0 ) + ) + ); + } + + set_transaction_headers(trx); + trx.sign( get_private_key( creator, "active" ), control->get_chain_id() ); + return push_transaction( trx ); + } + + action_result regproducer( const account_name& acnt, int params_fixture = 1 ) { + action_result r = push_action( acnt, "regproducer"_n, mvo() + ("producer", acnt ) + ("producer_key", get_public_key( acnt, "active" ) ) + ("url", "" ) + ("location", 0 ) + ); + BOOST_CHECK_EQUAL( success(), r); + return r; + } + + action_result unstake(const account_name& from, const account_name& to, const asset& net, const asset& cpu){ + return push_action(name(from), "undelegatebw"_n, mvo() + ("from", from) + ("receiver", to) + ("unstake_net_quantity", net ) + ("unstake_cpu_quantity", cpu ) + ); + } + + action_result buyram( const account_name& payer, account_name receiver, const asset& eosin ) { + return push_action( payer, "buyram"_n, mvo()( "payer",payer)("receiver",receiver)("quant",eosin) ); + } + + vector active_and_vote_producers() { + //stake more than 15% of total EOS supply to activate chain + transfer( name("eosio"), name("alice1111111"), core_from_string("650000000.0000"), name("eosio") ); + BOOST_CHECK_EQUAL( success(), stake( name("alice1111111"), name("alice1111111"), core_from_string("300000000.0000"), core_from_string("300000000.0000") ) ); + + // create accounts {defproducera, defproducerb, ..., defproducerz} and register as producers + std::vector producer_names; + { + producer_names.reserve('z' - 'a' + 1); + const std::string root("defproducer"); + for ( char c = 'a'; c < 'a'+21; ++c ) { + producer_names.emplace_back(root + std::string(1, c)); + } + setup_producer_accounts(producer_names); + for (const auto& p: producer_names) { + + BOOST_CHECK_EQUAL( success(), regproducer(p) ); + } + } + produce_blocks( 250); + + auto trace_auth = TESTER::push_action(config::system_account_name, updateauth::get_name(), config::system_account_name, mvo() + ("account", name(config::system_account_name).to_string()) + ("permission", name(config::active_name).to_string()) + ("parent", name(config::owner_name).to_string()) + ("auth", authority(1, {key_weight{get_public_key( config::system_account_name, "active" ), 1}}, { + permission_level_weight{{config::system_account_name, config::eosio_code_name}, 1}, + permission_level_weight{{config::producers_account_name, config::active_name}, 1} + } + )) + ); + BOOST_CHECK_EQUAL(transaction_receipt::executed, trace_auth->receipt->status); + + //vote for producers + { + transfer( config::system_account_name, name("alice1111111"), core_from_string("100000000.0000"), config::system_account_name ); + BOOST_CHECK_EQUAL(success(), stake( name("alice1111111"), core_from_string("30000000.0000"), core_from_string("30000000.0000") ) ); + BOOST_CHECK_EQUAL(success(), buyram( name("alice1111111"), name("alice1111111"), core_from_string("30000000.0000") ) ); + BOOST_CHECK_EQUAL(success(), push_action("alice1111111"_n, "voteproducer"_n, mvo() + ("voter", "alice1111111") + ("proxy", name(0).to_string()) + ("producers", vector(producer_names.begin(), producer_names.begin()+21)) + ) + ); + } + produce_blocks( 250 ); + + auto producer_keys = control->head_block_state()->active_schedule.producers; + BOOST_CHECK_EQUAL( 21, producer_keys.size() ); + BOOST_CHECK_EQUAL( name("defproducera"), producer_keys[0].producer_name ); + + return producer_names; + } + + action_result buyrex(const name& from, const asset& amount){ + return push_action(name(from), "buyrex"_n, mvo() + ("from", from) + ("amount", amount) + ); + } + + abi_serializer abi_ser; + abi_serializer token_abi_ser; +}; + +BOOST_AUTO_TEST_SUITE(chain_plugin_tests) + +BOOST_FIXTURE_TEST_CASE(account_results_total_resources_test, chain_plugin_tester) { try { + + produce_blocks(10); + setup_system_accounts(); + produce_blocks(); + create_account_with_resources("alice1111111"_n, config::system_account_name); + //stake more than 15% of total EOS supply to activate chain + transfer( name("eosio"), name("alice1111111"), core_from_string("650000000.0000"), name("eosio") ); + + read_only::get_account_results results = get_account_info(name("alice1111111")); + BOOST_CHECK(results.total_resources.get_type() != fc::variant::type_id::null_type); + BOOST_CHECK_EQUAL(core_from_string("10.0000"), results.total_resources["net_weight"].as()); + BOOST_CHECK_EQUAL(core_from_string("10.0000"), results.total_resources["cpu_weight"].as()); + BOOST_CHECK_EQUAL(results.total_resources["ram_bytes"].as_int64() > 0, true); + +} FC_LOG_AND_RETHROW() } + +BOOST_FIXTURE_TEST_CASE(account_results_self_delegated_bandwidth_test, chain_plugin_tester) { try { + + produce_blocks(10); + setup_system_accounts(); + produce_blocks(); + const asset nstake = core_from_string("1.0000"); + const asset cstake = core_from_string("2.0000"); + create_account_with_resources("alice1111111"_n, config::system_account_name, core_from_string("1.0000"), false); + BOOST_CHECK_EQUAL(success(), stake(config::system_account_name, name("alice1111111"), nstake, cstake)); + + read_only::get_account_results results = get_account_info(name("alice1111111")); + BOOST_CHECK(results.total_resources.get_type() != fc::variant::type_id::null_type); + BOOST_CHECK_EQUAL(core_from_string("11.0000"), results.total_resources["net_weight"].as()); + BOOST_CHECK_EQUAL(core_from_string("12.0000"), results.total_resources["cpu_weight"].as()); + + //self delegate bandwidth + transfer( name("eosio"), name("alice1111111"), core_from_string("650000000.0000"), name("eosio") ); + BOOST_CHECK_EQUAL(success(), stake(name("alice1111111"), name("alice1111111"), nstake, cstake)); + + results = get_account_info(name("alice1111111")); + BOOST_CHECK(results.self_delegated_bandwidth.get_type() != fc::variant::type_id::null_type); + BOOST_CHECK_EQUAL(core_from_string("1.0000"), results.self_delegated_bandwidth["net_weight"].as()); + BOOST_CHECK_EQUAL(core_from_string("2.0000"), results.self_delegated_bandwidth["cpu_weight"].as()); + + BOOST_CHECK(results.total_resources.get_type() != fc::variant::type_id::null_type); + BOOST_CHECK_EQUAL(core_from_string("12.0000"), results.total_resources["net_weight"].as()); + BOOST_CHECK_EQUAL(core_from_string("14.0000"), results.total_resources["cpu_weight"].as()); + +} FC_LOG_AND_RETHROW() } + +BOOST_FIXTURE_TEST_CASE(account_results_refund_request_test, chain_plugin_tester) { try { + + produce_blocks(10); + setup_system_accounts(); + produce_blocks(); + + setup_producer_accounts({"producer1111"_n}); + regproducer("producer1111"_n); + + read_only::get_account_results results = get_account_info(name("producer1111")); + BOOST_CHECK(results.total_resources.get_type() != fc::variant::type_id::null_type); + BOOST_CHECK_EQUAL(core_from_string("80.0000"), results.total_resources["net_weight"].as()); + + //cross 15 percent threshold + { + signed_transaction trx; + set_transaction_headers(trx); + + trx.actions.emplace_back( get_action( config::system_account_name, "delegatebw"_n, + vector{{config::system_account_name, config::active_name}}, + mvo() + ("from", name{config::system_account_name}) + ("receiver", "producer1111") + ("stake_net_quantity", core_from_string("150000000.0000") ) + ("stake_cpu_quantity", core_from_string("0.0000") ) + ("transfer", 1 ) + ) + ); + trx.actions.emplace_back( get_action( config::system_account_name, "voteproducer"_n, + vector{{"producer1111"_n, config::active_name}}, + mvo() + ("voter", "producer1111") + ("proxy", name(0).to_string()) + ("producers", vector(1, "producer1111"_n)) + ) + ); + + set_transaction_headers(trx); + trx.sign( get_private_key( config::system_account_name, "active" ), control->get_chain_id() ); + trx.sign( get_private_key( "producer1111"_n, "active" ), control->get_chain_id() ); + push_transaction( trx ); + } + + results = get_account_info(name("producer1111")); + BOOST_CHECK_EQUAL(core_from_string("150000080.0000"), results.total_resources["net_weight"].as()); + + BOOST_CHECK_EQUAL(success(), unstake(name("producer1111"),name("producer1111"), core_from_string("150000000.0000"), core_from_string("0.0000"))); + results = get_account_info(name("producer1111")); + BOOST_CHECK(results.total_resources.get_type() != fc::variant::type_id::null_type); + BOOST_CHECK(results.refund_request.get_type() != fc::variant::type_id::null_type); + BOOST_CHECK_EQUAL(core_from_string("150000000.0000"), results.refund_request["net_amount"].as()); + BOOST_CHECK_EQUAL(core_from_string("80.0000"), results.total_resources["net_weight"].as()); + + +} FC_LOG_AND_RETHROW() } + +BOOST_FIXTURE_TEST_CASE(account_results_voter_info_test, chain_plugin_tester) { try { + + produce_blocks(10); + setup_system_accounts(); + + create_account_with_resources("alice1111111"_n, config::system_account_name, core_from_string("1.0000"), false); + + active_and_vote_producers(); + read_only::get_account_results results = get_account_info(name("alice1111111")); + + BOOST_CHECK(results.voter_info.get_type() != fc::variant::type_id::null_type); + BOOST_CHECK_EQUAL(21, results.voter_info["producers"].size()); + +} FC_LOG_AND_RETHROW() } + +BOOST_FIXTURE_TEST_CASE(account_results_rex_info_test, chain_plugin_tester) { try { + + produce_blocks(10); + setup_system_accounts(); + + create_account_with_resources("alice1111111"_n, config::system_account_name, core_from_string("1.0000"), false); + + //stake more than 15% of total EOS supply to activate chain + transfer( name("eosio"), name("alice1111111"), core_from_string("650000000.0000"), name("eosio") ); + deposit(name("alice1111111"), core_from_string("1000.0000")); + BOOST_CHECK_EQUAL( success(), buyrex(name("alice1111111"), core_from_string("100.0000")) ); + + read_only::get_account_results results = get_account_info(name("alice1111111")); + BOOST_CHECK(results.rex_info.get_type() != fc::variant::type_id::null_type); + BOOST_CHECK_EQUAL(core_from_string("100.0000"), results.rex_info["vote_stake"].as()); + //BOOST_CHECK_EQUAL(0, results.rex_info["matured_rex"]); + +} FC_LOG_AND_RETHROW() } + +BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/plugins/db_size_api_plugin/db_size_api_plugin.cpp b/plugins/db_size_api_plugin/db_size_api_plugin.cpp index 1f5cff033b8..0fab2bbc676 100644 --- a/plugins/db_size_api_plugin/db_size_api_plugin.cpp +++ b/plugins/db_size_api_plugin/db_size_api_plugin.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace eosio { @@ -8,11 +9,11 @@ static appbase::abstract_plugin& _db_size_api_plugin = app().register_plugin(body); \ INVOKE \ cb(http_response_code, fc::variant(result)); \ } catch (...) { \ @@ -26,13 +27,12 @@ using namespace eosio; void db_size_api_plugin::plugin_startup() { app().get_plugin().add_api({ - CALL(db_size, this, get, - INVOKE_R_V(this, get), 200), + CALL_WITH_400(db_size, this, get, INVOKE_R_V(this, get), 200), + CALL_WITH_400(db_size, this, get_reversible, INVOKE_R_V(this, get_reversible), 200), }); } -db_size_stats db_size_api_plugin::get() { - const chainbase::database& db = app().get_plugin().chain().db(); +db_size_stats db_size_api_plugin::get_db_stats(const chainbase::database& db) { db_size_stats ret; ret.free_bytes = db.get_segment_manager()->get_free_memory(); @@ -46,6 +46,14 @@ db_size_stats db_size_api_plugin::get() { return ret; } +db_size_stats db_size_api_plugin::get() { + return get_db_stats(app().get_plugin().chain().db()); +} + +db_size_stats db_size_api_plugin::get_reversible() { + return get_db_stats(app().get_plugin().chain().reversible_db()); +} + #undef INVOKE_R_V #undef CALL diff --git a/plugins/db_size_api_plugin/include/eosio/db_size_api_plugin/db_size_api_plugin.hpp b/plugins/db_size_api_plugin/include/eosio/db_size_api_plugin/db_size_api_plugin.hpp index 76495b23381..aebe0af25b7 100644 --- a/plugins/db_size_api_plugin/include/eosio/db_size_api_plugin/db_size_api_plugin.hpp +++ b/plugins/db_size_api_plugin/include/eosio/db_size_api_plugin/db_size_api_plugin.hpp @@ -38,8 +38,10 @@ class db_size_api_plugin : public plugin { void plugin_shutdown() {} db_size_stats get(); + db_size_stats get_reversible(); private: + db_size_stats get_db_stats(const chainbase::database& ); }; } diff --git a/plugins/faucet_testnet_plugin/CMakeLists.txt b/plugins/faucet_testnet_plugin/CMakeLists.txt deleted file mode 100644 index 8dda35f979e..00000000000 --- a/plugins/faucet_testnet_plugin/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -file(GLOB HEADERS "include/eosio/faucet_testnet_plugin/*.hpp") -add_library( faucet_testnet_plugin - faucet_testnet_plugin.cpp - ${HEADERS} ) - -target_link_libraries( faucet_testnet_plugin appbase fc http_plugin chain_plugin ) -target_include_directories( faucet_testnet_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) diff --git a/plugins/faucet_testnet_plugin/faucet_testnet_plugin.cpp b/plugins/faucet_testnet_plugin/faucet_testnet_plugin.cpp deleted file mode 100644 index 592a799aad3..00000000000 --- a/plugins/faucet_testnet_plugin/faucet_testnet_plugin.cpp +++ /dev/null @@ -1,323 +0,0 @@ -#include -#include - -#include -#include -#include -#include - -#include -#include -#include - -#include - -namespace eosio { namespace detail { - struct faucet_testnet_empty {}; - - struct faucet_testnet_keys { - std::string owner; - std::string active; - }; - - struct faucet_testnet_create_account_params { - std::string account; - faucet_testnet_keys keys; - }; - - struct faucet_testnet_create_account_alternates_response { - std::vector alternates; - std::string message; - }; - - struct faucet_testnet_create_account_rate_limited_response { - std::string message; - }; -}} - -FC_REFLECT(eosio::detail::faucet_testnet_empty, ); -FC_REFLECT(eosio::detail::faucet_testnet_keys, (owner)(active)); -FC_REFLECT(eosio::detail::faucet_testnet_create_account_params, (account)(keys)); -FC_REFLECT(eosio::detail::faucet_testnet_create_account_alternates_response, (alternates)(message)); -FC_REFLECT(eosio::detail::faucet_testnet_create_account_rate_limited_response, (message)); - -namespace eosio { - -static appbase::abstract_plugin& _faucet_testnet_plugin = app().register_plugin(); - -using namespace eosio::chain; -using public_key_type = chain::public_key_type; -using key_pair = std::pair; -using results_pair = std::pair; - -#define CALL(api_name, api_handle, call_name, invoke_cb) \ -{std::string("/v1/" #api_name "/" #call_name), \ - [this](string, string body, url_response_callback response_cb) mutable { \ - try { \ - if (body.empty()) body = "{}"; \ - const auto result = api_handle->invoke_cb(body); \ - response_cb(result.first, fc::variant(result.second)); \ - } catch (...) { \ - http_plugin::handle_exception(#api_name, #call_name, body, response_cb); \ - } \ - }} - -struct faucet_testnet_plugin_impl { - struct create_faucet_account_alternate_results { - std::vector alternates; - }; - - faucet_testnet_plugin_impl(appbase::application& app) - : _app(app) - , _timer{app.get_io_service()} - { - } - - void timer_fired() { - _blocking_accounts = false; - } - - enum http_return_codes { - account_created = 201, - conflict_with_alternates = 409, - too_many_requests = 429 - }; - - class extension { - - public: - explicit extension(uint32_t capacity) - : max_capacity(capacity) - { - } - - bool increment() { - bool incremented = false; - // start incrementing from the end back to the begining - auto rev_itr = ext.rbegin(); - for (; rev_itr != ext.rend(); ++rev_itr) { - // done once a character is incremented - if (increment(*rev_itr)) { - std::string temp(1, *rev_itr); - incremented = true; - break; - } - } - - const bool add_char = (rev_itr == ext.rend()); - - if (incremented) - return true; - else if (add_char && ext.length() >= max_capacity) { - return false; - } - - if (!ext.empty()) { - do { - --rev_itr; - *rev_itr = init_char; - std::string temp(1, *rev_itr); - } while(rev_itr != ext.rbegin()); - } - - if (add_char) { - ext.push_back(init_char); - } - return true; - } - - std::string to_string() { - return ext; - } - - // assuming 13 characters, which will mean inefficiencies if forced to search with 13th char, since only 1 through j is valid there - static const uint32_t max_allowed_name_length = 13; - - private: - // NOTE: code written expecting struct name to be using charmap of ".12345abcdefghijklmnopqrstuvwxyz" - bool increment(char& c) { - std::string temp(1, c); - if (c >= '1' && c < '5') - ++c; - else if (c == '5') - c = 'a'; - else if (c >= 'a' && c < 'z') - ++c; - else - return false; - - temp.assign(1, c); - return true; - } - - const uint32_t max_capacity; - std::string ext; - const char init_char = '1'; - }; - - results_pair find_alternates(const std::string& new_account_name) { - std::vector names; - std::string suggestion = new_account_name; - while (names.size() < _default_create_alternates_to_return && - !suggestion.empty()) { - const int32_t extension_capacity = extension::max_allowed_name_length - suggestion.length(); - if (extension_capacity > 0) { - extension ext(extension_capacity); - while(names.size() < _default_create_alternates_to_return && ext.increment()) { - chain::account_name alt; - // not externalizing all the name struct encoding, so need to handle cases where we - // construct an invalid encoded name - try { - alt = suggestion + ext.to_string(); - } catch (const fc::assert_exception& ) { - continue; - } - const account_object* const obj = database().find(alt); - if (obj == nullptr) { - names.push_back(alt); - } - } - } - // drop the trailing character and try again - suggestion.pop_back(); - } - - const eosio::detail::faucet_testnet_create_account_alternates_response response{ - names, "Account name is already in use."}; - return { conflict_with_alternates, fc::variant(response) }; - } - - results_pair create_account(const std::string& new_account_name, const fc::crypto::public_key& owner_pub_key, const fc::crypto::public_key& active_pub_key) { - - auto creating_account = database().find(_create_account_name); - EOS_ASSERT(creating_account != nullptr, transaction_exception, - "To create account using the faucet, must already have created account \"${a}\"",("a",_create_account_name)); - - auto existing_account = database().find(new_account_name); - if (existing_account != nullptr) - { - return find_alternates(new_account_name); - } - - if (_blocking_accounts) - { - eosio::detail::faucet_testnet_create_account_rate_limited_response response{ - "Rate limit exceeded, the max is 1 request per " + fc::to_string(_create_interval_msec) + - " milliseconds. Come back later."}; - return std::make_pair(too_many_requests, fc::variant(response)); - } - - chain::chain_id_type chainid; - auto& plugin = _app.get_plugin(); - plugin.get_chain_id(chainid); - controller& cc = plugin.chain(); - - signed_transaction trx; - auto memo = fc::variant(fc::time_point::now()).as_string() + " " + fc::variant(fc::time_point::now().time_since_epoch()).as_string(); - - //create "A" account - auto owner_auth = chain::authority{1, {{owner_pub_key, 1}}, {}}; - auto active_auth = chain::authority{1, {{active_pub_key, 1}}, {}}; - auto recovery_auth = chain::authority{1, {}, {{{_create_account_name, "active"}, 1}}}; - - trx.actions.emplace_back(vector{{_create_account_name,"active"}}, - newaccount{_create_account_name, new_account_name, owner_auth, active_auth}); - - trx.expiration = cc.head_block_time() + fc::seconds(30); - trx.set_reference_block(cc.head_block_id()); - trx.sign(_create_account_private_key, chainid); - - try { - cc.push_transaction( std::make_shared(trx) ); - } catch (const account_name_exists_exception& ) { - // another transaction ended up adding the account, so look for alternates - return find_alternates(new_account_name); - } - - _blocking_accounts = true; - _timer.expires_from_now(boost::posix_time::microseconds(_create_interval_msec * 1000)); - _timer.async_wait(std::bind(&faucet_testnet_plugin_impl::timer_fired, this)); - - return std::make_pair(account_created, fc::variant(eosio::detail::faucet_testnet_empty())); - } - - results_pair create_faucet_account(const std::string& body) { - const eosio::detail::faucet_testnet_create_account_params params = fc::json::from_string(body).as(); - return create_account(params.account, fc::crypto::public_key(params.keys.owner), fc::crypto::public_key(params.keys.active)); - } - - const chainbase::database& database() { - static const chainbase::database* db = nullptr; - if (db == nullptr) - db = &_app.get_plugin().chain().db(); - - return *db; - } - - appbase::application& _app; - boost::asio::deadline_timer _timer; - bool _blocking_accounts = false; - - static const uint32_t _default_create_interval_msec; - uint32_t _create_interval_msec; - static const uint32_t _default_create_alternates_to_return; - static const std::string _default_create_account_name; - chain::account_name _create_account_name; - static const key_pair _default_key_pair; - fc::crypto::private_key _create_account_private_key; - public_key_type _create_account_public_key; -}; - -const uint32_t faucet_testnet_plugin_impl::_default_create_interval_msec = 1000; -const uint32_t faucet_testnet_plugin_impl::_default_create_alternates_to_return = 3; -const std::string faucet_testnet_plugin_impl::_default_create_account_name = "faucet"; -// defaults to the public/private key of init accounts in private testnet genesis.json -const key_pair faucet_testnet_plugin_impl::_default_key_pair = {"EOS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV", "5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3"}; - - -faucet_testnet_plugin::faucet_testnet_plugin() -: my(new faucet_testnet_plugin_impl(app())) -{ -} - -faucet_testnet_plugin::~faucet_testnet_plugin() {} - -void faucet_testnet_plugin::set_program_options(options_description&, options_description& cfg) { - cfg.add_options() - ("faucet-create-interval-ms", bpo::value()->default_value(faucet_testnet_plugin_impl::_default_create_interval_msec), - "Time to wait, in milliseconds, between creating next faucet created account.") - ("faucet-name", bpo::value()->default_value(faucet_testnet_plugin_impl::_default_create_account_name), - "Name to use as creator for faucet created accounts.") - ("faucet-private-key", boost::program_options::value()->default_value(fc::json::to_string(faucet_testnet_plugin_impl::_default_key_pair)), - "[public key, WIF private key] for signing for faucet creator account") - ; -} - -void faucet_testnet_plugin::plugin_initialize(const variables_map& options) { - try { - my->_create_interval_msec = options.at( "faucet-create-interval-ms" ).as(); - my->_create_account_name = options.at( "faucet-name" ).as(); - - auto faucet_key_pair = fc::json::from_string( options.at( "faucet-private-key" ).as()).as(); - my->_create_account_public_key = public_key_type( faucet_key_pair.first ); - ilog( "Public Key: ${public}", ("public", my->_create_account_public_key)); - fc::crypto::private_key private_key( faucet_key_pair.second ); - my->_create_account_private_key = std::move( private_key ); - } FC_LOG_AND_RETHROW() -} - -void faucet_testnet_plugin::plugin_startup() { - app().get_plugin().add_api({ - CALL(faucet, my, create_account, faucet_testnet_plugin_impl::create_faucet_account ) - }); -} - -void faucet_testnet_plugin::plugin_shutdown() { - try { - my->_timer.cancel(); - } catch(fc::exception& e) { - edump((e.to_detail_string())); - } -} - -} diff --git a/plugins/faucet_testnet_plugin/include/eosio/faucet_testnet_plugin/faucet_testnet_plugin.hpp b/plugins/faucet_testnet_plugin/include/eosio/faucet_testnet_plugin/faucet_testnet_plugin.hpp deleted file mode 100644 index 237347df985..00000000000 --- a/plugins/faucet_testnet_plugin/include/eosio/faucet_testnet_plugin/faucet_testnet_plugin.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once -#include -#include - -namespace eosio { - -using namespace appbase; - -class faucet_testnet_plugin : public appbase::plugin { -public: - faucet_testnet_plugin(); - ~faucet_testnet_plugin(); - - APPBASE_PLUGIN_REQUIRES((http_plugin)) - virtual void set_program_options(options_description&, options_description& cfg) override; - - void plugin_initialize(const variables_map& options); - void plugin_startup(); - void plugin_shutdown(); - -private: - std::unique_ptr my; -}; - -} diff --git a/plugins/history_api_plugin/history_api_plugin.cpp b/plugins/history_api_plugin/history_api_plugin.cpp index b5155c68f7e..c435371f42e 100644 --- a/plugins/history_api_plugin/history_api_plugin.cpp +++ b/plugins/history_api_plugin/history_api_plugin.cpp @@ -15,19 +15,19 @@ history_api_plugin::~history_api_plugin(){} void history_api_plugin::set_program_options(options_description&, options_description&) {} void history_api_plugin::plugin_initialize(const variables_map&) {} -#define CALL(api_name, api_handle, api_namespace, call_name) \ +#define CALL_WITH_400(api_name, api_handle, api_namespace, call_name, params_type) \ {std::string("/v1/" #api_name "/" #call_name), \ [api_handle](string, string body, url_response_callback cb) mutable { \ try { \ - if (body.empty()) body = "{}"; \ - fc::variant result( api_handle.call_name(fc::json::from_string(body).as()) ); \ + auto params = parse_params(body);\ + fc::variant result( api_handle.call_name( std::move(params) ) ); \ cb(200, std::move(result)); \ } catch (...) { \ http_plugin::handle_exception(#api_name, #call_name, body, cb); \ } \ }} -#define CHAIN_RO_CALL(call_name) CALL(history, ro_api, history_apis::read_only, call_name) +#define CHAIN_RO_CALL(call_name, params_type) CALL_WITH_400(history, ro_api, history_apis::read_only, call_name, params_type) //#define CHAIN_RW_CALL(call_name) CALL(history, rw_api, history_apis::read_write, call_name) void history_api_plugin::plugin_startup() { @@ -37,10 +37,10 @@ void history_api_plugin::plugin_startup() { app().get_plugin().add_api({ // CHAIN_RO_CALL(get_transaction), - CHAIN_RO_CALL(get_actions), - CHAIN_RO_CALL(get_transaction), - CHAIN_RO_CALL(get_key_accounts), - CHAIN_RO_CALL(get_controlled_accounts) + CHAIN_RO_CALL(get_actions, http_params_types::params_required), + CHAIN_RO_CALL(get_transaction, http_params_types::params_required), + CHAIN_RO_CALL(get_key_accounts, http_params_types::params_required), + CHAIN_RO_CALL(get_controlled_accounts, http_params_types::params_required) }); } diff --git a/plugins/history_plugin/history_plugin.cpp b/plugins/history_plugin/history_plugin.cpp index 05dce34f8b6..c655aac8303 100644 --- a/plugins/history_plugin/history_plugin.cpp +++ b/plugins/history_plugin/history_plugin.cpp @@ -44,14 +44,14 @@ namespace eosio { struct by_action_sequence_num; struct by_account_action_seq; - struct by_trx_id; + struct by_trx_id_act_seq; using action_history_index = chainbase::shared_multi_index_container< action_history_object, indexed_by< ordered_unique, member>, ordered_unique, member>, - ordered_unique, + ordered_unique, composite_key< action_history_object, member, member @@ -141,7 +141,7 @@ namespace eosio { std::set filter_on; std::set filter_out; chain_plugin* chain_plug = nullptr; - fc::optional applied_transaction_connection; + std::optional applied_transaction_connection; bool filter(const action_trace& act) { bool pass_on = false; @@ -226,26 +226,26 @@ namespace eosio { void on_system_action( const action_trace& at ) { auto& chain = chain_plug->chain(); chainbase::database& db = const_cast( chain.db() ); // Override read-only access to state DB (highly unrecommended practice!) - if( at.act.name == N(newaccount) ) + if( at.act.name == "newaccount"_n ) { const auto create = at.act.data_as(); - add(db, create.owner.keys, create.name, N(owner)); - add(db, create.owner.accounts, create.name, N(owner)); - add(db, create.active.keys, create.name, N(active)); - add(db, create.active.accounts, create.name, N(active)); + add(db, create.owner.keys, create.name, "owner"_n); + add(db, create.owner.accounts, create.name, "owner"_n); + add(db, create.active.keys, create.name, "active"_n); + add(db, create.active.accounts, create.name, "active"_n); } - else if( at.act.name == N(updateauth) ) + else if( at.act.name == "updateauth"_n ) { const auto update = at.act.data_as(); - remove(db, update.account, update.permission); + remove(db, update.account, update.permission); remove(db, update.account, update.permission); add(db, update.auth.keys, update.account, update.permission); add(db, update.auth.accounts, update.account, update.permission); } - else if( at.act.name == N(deleteauth) ) + else if( at.act.name == "deleteauth"_n ) { const auto del = at.act.data_as(); - remove(db, del.account, del.permission); + remove(db, del.account, del.permission); remove(db, del.account, del.permission); } } @@ -258,9 +258,10 @@ namespace eosio { db.create( [&]( auto& aho ) { auto ps = fc::raw::pack_size( at ); - aho.packed_action_trace.resize(ps); - datastream ds( aho.packed_action_trace.data(), ps ); - fc::raw::pack( ds, at ); + aho.packed_action_trace.resize_and_fill(ps, [&at](char* data, std::size_t size) { + fc::datastream ds( data, size ); + fc::raw::pack( ds, at ); + }); aho.action_sequence_num = at.receipt->global_sequence; aho.block_num = chain.head_block_num() + 1; aho.block_time = chain.pending_block_time(); @@ -351,7 +352,7 @@ namespace eosio { db.add_index(); my->applied_transaction_connection.emplace( - chain.applied_transaction.connect( [&]( std::tuple t ) { + chain.applied_transaction.connect( [&]( std::tuple t ) { my->on_applied_transaction( std::get<0>(t) ); } )); } FC_LOG_AND_RETHROW() @@ -462,7 +463,7 @@ namespace eosio { }; const auto& db = chain.db(); - const auto& idx = db.get_index(); + const auto& idx = db.get_index(); auto itr = idx.lower_bound( boost::make_tuple( input_id ) ); bool in_history = (itr != idx.end() && txn_id_matched(itr->trx_id) ); @@ -491,18 +492,24 @@ namespace eosio { auto blk = chain.fetch_block_by_number( result.block_num ); if( blk || chain.is_building_block() ) { - const vector& receipts = blk ? blk->transactions : chain.get_pending_trx_receipts(); + const auto& receipts = blk ? blk->transactions : chain.get_pending_trx_receipts(); for (const auto &receipt: receipts) { - if (receipt.trx.contains()) { - auto &pt = receipt.trx.get(); + if (std::holds_alternative(receipt.trx)) { + auto &pt = std::get(receipt.trx); if (pt.id() == result.id) { fc::mutable_variant_object r("receipt", receipt); - r("trx", chain.to_variant_with_abi(pt.get_signed_transaction(), abi_serializer::create_yield_function( abi_serializer_max_time ))); + fc::variant v = chain.to_variant_with_abi(pt.get_transaction(), abi_serializer::create_yield_function( abi_serializer_max_time )); + fc::mutable_variant_object tmp(v.get_object()); + const auto* sigs = pt.get_signatures(); + tmp("signatures", sigs != nullptr ? *sigs : vector()); + const auto* context_free_data = pt.get_context_free_data(); + tmp("context_free_data", context_free_data != nullptr ? *context_free_data : vector()); + r("trx", std::move(tmp) ); result.trx = move(r); break; } } else { - auto &id = receipt.trx.get(); + auto &id = std::get(receipt.trx); if (id == result.id) { fc::mutable_variant_object r("receipt", receipt); result.trx = move(r); @@ -516,8 +523,8 @@ namespace eosio { bool found = false; if (blk) { for (const auto& receipt: blk->transactions) { - if (receipt.trx.contains()) { - auto& pt = receipt.trx.get(); + if (std::holds_alternative(receipt.trx)) { + auto& pt = std::get(receipt.trx); const auto& id = pt.id(); if( txn_id_matched(id) ) { result.id = id; @@ -525,13 +532,19 @@ namespace eosio { result.block_num = *p.block_num_hint; result.block_time = blk->timestamp; fc::mutable_variant_object r("receipt", receipt); - r("trx", chain.to_variant_with_abi(pt.get_signed_transaction(), abi_serializer::create_yield_function( abi_serializer_max_time ))); + fc::variant v = chain.to_variant_with_abi(pt.get_transaction(), abi_serializer::create_yield_function( abi_serializer_max_time )); + fc::mutable_variant_object tmp(v.get_object()); + const auto* sigs = pt.get_signatures(); + tmp("signatures", sigs != nullptr ? *sigs : vector()); + const auto* context_free_data = pt.get_context_free_data(); + tmp("context_free_data", context_free_data != nullptr ? *context_free_data : vector()); + r("trx", std::move(tmp) ); result.trx = move(r); found = true; break; } } else { - auto& id = receipt.trx.get(); + auto& id = std::get(receipt.trx); if( txn_id_matched(id) ) { result.id = id; result.last_irreversible_block = chain.last_irreversible_block_num(); diff --git a/plugins/history_plugin/include/eosio/history_plugin/history_plugin.hpp b/plugins/history_plugin/include/eosio/history_plugin/history_plugin.hpp index 0bbeda361be..365d902f428 100644 --- a/plugins/history_plugin/include/eosio/history_plugin/history_plugin.hpp +++ b/plugins/history_plugin/include/eosio/history_plugin/history_plugin.hpp @@ -10,7 +10,6 @@ namespace eosio { using std::shared_ptr; using namespace appbase; using chain::name; - using fc::optional; using chain::uint128_t; typedef shared_ptr history_ptr; @@ -27,9 +26,9 @@ class read_only { struct get_actions_params { - chain::account_name account_name; - optional pos; /// a absolute sequence positon -1 is the end/last action - optional offset; ///< the number of actions relative to pos, negative numbers return [pos-offset,pos), positive numbers return [pos,pos+offset) + chain::account_name account_name; + std::optional pos; /// a absolute sequence positon -1 is the end/last action + std::optional offset; ///< the number of actions relative to pos, negative numbers return [pos-offset,pos), positive numbers return [pos,pos+offset) }; struct ordered_action_result { @@ -43,7 +42,7 @@ class read_only { struct get_actions_result { vector actions; uint32_t last_irreversible_block; - optional time_limit_exceeded_error; + std::optional time_limit_exceeded_error; }; @@ -52,7 +51,7 @@ class read_only { struct get_transaction_params { string id; - optional block_num_hint; + std::optional block_num_hint; }; struct get_transaction_result { diff --git a/plugins/history_plugin/include/eosio/history_plugin/public_key_history_object.hpp b/plugins/history_plugin/include/eosio/history_plugin/public_key_history_object.hpp index 052f74c5a54..79b10e36d90 100644 --- a/plugins/history_plugin/include/eosio/history_plugin/public_key_history_object.hpp +++ b/plugins/history_plugin/include/eosio/history_plugin/public_key_history_object.hpp @@ -20,7 +20,7 @@ class public_key_history_object : public chainbase::object > >, - ordered_unique, + ordered_unique, composite_key< public_key_history_object, member, member, diff --git a/plugins/http_client_plugin/http_client_plugin.cpp b/plugins/http_client_plugin/http_client_plugin.cpp index 60811ff44ba..fa052d7ebf8 100644 --- a/plugins/http_client_plugin/http_client_plugin.cpp +++ b/plugins/http_client_plugin/http_client_plugin.cpp @@ -33,15 +33,27 @@ void http_client_plugin::plugin_initialize(const variables_map& options) { EOS_ASSERT( boost::algorithm::starts_with( pem_str, "-----BEGIN CERTIFICATE-----\n" ), chain::invalid_http_client_root_cert, "File does not appear to be a PEM encoded certificate" ); + } catch ( const std::bad_alloc& ) { + throw; + } catch ( const boost::interprocess::bad_alloc& ) { + throw; } catch ( const fc::exception& e ) { elog( "Failed to read PEM ${f} : ${e}", ("f", root_pem)( "e", e.to_detail_string())); + } catch ( const std::exception& e ) { + elog( "Failed to read PEM ${f} : ${e}", ("f", root_pem)( "e", fc::std_exception_wrapper::from_current_exception(e).to_detail_string())); } } try { my->add_cert( pem_str ); + } catch ( const std::bad_alloc& ) { + throw; + } catch ( const boost::interprocess::bad_alloc& ) { + throw; } catch ( const fc::exception& e ) { elog( "Failed to read PEM : ${e} \n${pem}\n", ("pem", pem_str)( "e", e.to_detail_string())); + } catch ( const std::exception& e ) { + elog( "Failed to read PEM : ${e} \n${pem}\n", ("pem", pem_str)( "e", fc::std_exception_wrapper::from_current_exception(e).to_detail_string())); } } } diff --git a/plugins/http_plugin/http_plugin.cpp b/plugins/http_plugin/http_plugin.cpp index 40df1b01f5d..6a0eb134cb4 100644 --- a/plugins/http_plugin/http_plugin.cpp +++ b/plugins/http_plugin/http_plugin.cpp @@ -54,7 +54,7 @@ namespace eosio { static http_plugin_defaults current_http_plugin_defaults; - void http_plugin::set_defaults(const http_plugin_defaults config) { + void http_plugin::set_defaults(const http_plugin_defaults& config) { current_http_plugin_defaults = config; } @@ -131,10 +131,12 @@ namespace eosio { * virtualized wrapper for the various underlying connection functions needed in req/resp processng */ struct abstract_conn { - virtual ~abstract_conn() {} + virtual ~abstract_conn() = default; virtual bool verify_max_bytes_in_flight() = 0; + virtual bool verify_max_requests_in_flight() = 0; virtual void handle_exception() = 0; - virtual void send_response(std::string, int) = 0; + + virtual void send_response(std::optional body, int code) = 0; }; using abstract_conn_ptr = std::shared_ptr; @@ -169,6 +171,21 @@ namespace eosio { } catch(...) {} return 0; } + + /** + * Helper method to calculate the "in flight" size of a std::optional + * When the optional doesn't contain value, it will return the size of 0 + * + * @param o - the std::optional where T is typename + * @return in flight size of o + */ + template + static size_t in_flight_sizeof( const std::optional& o ) { + if( o ) { + return in_flight_sizeof( *o ); + } + return 0; + } } using websocket_server_type = websocketpp::server>; @@ -193,30 +210,32 @@ class http_plugin_impl : public std::enable_shared_from_this { // key -> priority, url_handler map url_handlers; - optional listen_endpoint; - string access_control_allow_origin; - string access_control_allow_headers; - string access_control_max_age; - bool access_control_allow_credentials = false; - size_t max_body_size{1024*1024}; + std::optional listen_endpoint; + string access_control_allow_origin; + string access_control_allow_headers; + string access_control_max_age; + bool access_control_allow_credentials = false; + size_t max_body_size{1024*1024}; websocket_server_type server; - uint16_t thread_pool_size = 2; - optional thread_pool; - std::atomic bytes_in_flight{0}; - size_t max_bytes_in_flight = 0; - fc::microseconds max_response_time{30*1000}; + uint16_t thread_pool_size = 2; + std::optional thread_pool; + std::atomic bytes_in_flight{0}; + std::atomic requests_in_flight{0}; + size_t max_bytes_in_flight = 0; + int32_t max_requests_in_flight = -1; + fc::microseconds max_response_time{30*1000}; - optional https_listen_endpoint; - string https_cert_chain; - string https_key; - https_ecdh_curve_t https_ecdh_curve = SECP384R1; + std::optional https_listen_endpoint; + string https_cert_chain; + string https_key; + https_ecdh_curve_t https_ecdh_curve = SECP384R1; websocket_server_tls_type https_server; #ifdef BOOST_ASIO_HAS_LOCAL_SOCKETS - optional unix_endpoint; + std::optional unix_endpoint; websocket_local_server_type unix_server; #endif @@ -242,7 +261,7 @@ class http_plugin_impl : public std::enable_shared_from_this { } } - ssl_context_ptr on_tls_init(websocketpp::connection_hdl hdl) { + ssl_context_ptr on_tls_init() { ssl_context_ptr ctx = websocketpp::lib::make_shared(asio::ssl::context::sslv23_server); try { @@ -332,25 +351,46 @@ class http_plugin_impl : public std::enable_shared_from_this { return true; } + template + void report_429_error( const T& con, const std::string & what) { + error_results::error_info ei; + ei.code = websocketpp::http::status_code::too_many_requests; + ei.name = "Busy"; + ei.what = what; + error_results results{websocketpp::http::status_code::too_many_requests, "Busy", ei}; + con->set_body( fc::json::to_string( results, fc::time_point::maximum() )); + con->set_status( websocketpp::http::status_code::too_many_requests ); + con->send_http_response(); + } + template bool verify_max_bytes_in_flight( const T& con ) { auto bytes_in_flight_size = bytes_in_flight.load(); if( bytes_in_flight_size > max_bytes_in_flight ) { fc_dlog( logger, "429 - too many bytes in flight: ${bytes}", ("bytes", bytes_in_flight_size) ); - error_results::error_info ei; - ei.code = websocketpp::http::status_code::too_many_requests; - ei.name = "Busy"; - ei.what = "Too many bytes in flight: " + std::to_string( bytes_in_flight_size ); - error_results results{websocketpp::http::status_code::too_many_requests, "Busy", ei}; - con->set_body( fc::json::to_string( results, fc::time_point::maximum() )); - con->set_status( websocketpp::http::status_code::too_many_requests ); - con->send_http_response(); + string what = "Too many bytes in flight: " + std::to_string( bytes_in_flight_size ) + ". Try again later.";; + report_429_error(con, what); return false; } return true; } + template + bool verify_max_requests_in_flight( const T& con ) { + if (max_requests_in_flight < 0) + return true; + + auto requests_in_flight_num = requests_in_flight.load(); + if( requests_in_flight_num > max_requests_in_flight ) { + fc_dlog( logger, "429 - too many requests in flight: ${requests}", ("requests", requests_in_flight_num) ); + string what = "Too many requests in flight: " + std::to_string( requests_in_flight_num ) + ". Try again later."; + report_429_error(con, what); + return false; + } + return true; + } + /** * child struct, implementing abstract connection for various underlying connection types * that ties it to an http_plugin_impl @@ -362,26 +402,37 @@ class http_plugin_impl : public std::enable_shared_from_this { abstract_conn_impl(detail::connection_ptr conn, http_plugin_impl_ptr impl) :_conn(std::move(conn)) ,_impl(std::move(impl)) - {} + { + _impl->requests_in_flight += 1; + } + + ~abstract_conn_impl() override { + _impl->requests_in_flight -= 1; + } // No copy constructor and no move abstract_conn_impl(const abstract_conn_impl&) = delete; abstract_conn_impl(abstract_conn_impl&&) = delete; abstract_conn_impl& operator=(const abstract_conn_impl&) = delete; - abstract_conn_impl& operator=(abstract_conn_impl&&) noexcept = default; - ~abstract_conn_impl() = default; + abstract_conn_impl& operator=(abstract_conn_impl&&) noexcept = default; bool verify_max_bytes_in_flight() override { return _impl->verify_max_bytes_in_flight(_conn); } + bool verify_max_requests_in_flight() override { + return _impl->verify_max_requests_in_flight(_conn); + } + void handle_exception()override { http_plugin_impl::handle_exception(_conn); } - void send_response(std::string body, int code) override { - _conn->set_body(std::move(body)); + void send_response(std::optional body, int code) override { + if( body ) { + _conn->set_body( std::move( *body ) ); + } _conn->set_status( websocketpp::http::status_code::value( code ) ); _conn->send_http_response(); } @@ -447,15 +498,15 @@ class http_plugin_impl : public std::enable_shared_from_this { * const accessor * @return const reference to the contained object */ - const T& operator* () const { + const T& obj() const { return _object; } /** - * mutable accessor (can be moved frmo) + * mutable accessor (can be moved from) * @return mutable reference to the contained object */ - T& operator* () { + T& obj() { return _object; } @@ -491,7 +542,7 @@ class http_plugin_impl : public std::enable_shared_from_this { return; } - url_response_callback wrapped_then = [tracked_b, then=std::move(then)](int code, fc::variant resp) { + url_response_callback wrapped_then = [tracked_b, then=std::move(then)](int code, std::optional resp) { then(code, std::move(resp)); }; @@ -500,7 +551,7 @@ class http_plugin_impl : public std::enable_shared_from_this { app().post( priority, [next_ptr, conn=std::move(conn), r=std::move(r), tracked_b, wrapped_then=std::move(wrapped_then)]() mutable { try { // call the `next` url_handler and wrap the response handler - (*next_ptr)( std::move( r ), std::move(*(*tracked_b)), std::move(wrapped_then)) ; + (*next_ptr)( std::move( r ), std::move(tracked_b->obj()), std::move(wrapped_then)) ; } catch( ... ) { conn->handle_exception(); } @@ -516,7 +567,7 @@ class http_plugin_impl : public std::enable_shared_from_this { * @return the constructed internal_url_handler */ static detail::internal_url_handler make_http_thread_url_handler(url_handler next) { - return [next=std::move(next)]( detail::abstract_conn_ptr conn, string r, string b, url_response_callback then ) { + return [next=std::move(next)]( const detail::abstract_conn_ptr& conn, string r, string b, url_response_callback then ) { try { next(std::move(r), std::move(b), std::move(then)); } catch( ... ) { @@ -533,8 +584,8 @@ class http_plugin_impl : public std::enable_shared_from_this { * @return lambda suitable for url_response_callback */ template - auto make_http_response_handler( detail::abstract_conn_ptr abstract_conn_ptr) { - return [my=shared_from_this(), abstract_conn_ptr]( int code, fc::variant response ) { + auto make_http_response_handler( const detail::abstract_conn_ptr& abstract_conn_ptr) { + return [my=shared_from_this(), abstract_conn_ptr]( int code, std::optional response ) { auto tracked_response = make_in_flight(std::move(response), my); if (!abstract_conn_ptr->verify_max_bytes_in_flight()) { return; @@ -544,9 +595,13 @@ class http_plugin_impl : public std::enable_shared_from_this { boost::asio::post( my->thread_pool->get_executor(), [my, abstract_conn_ptr, code, tracked_response=std::move(tracked_response)]() { try { - std::string json = fc::json::to_string( *(*tracked_response), fc::time_point::now() + my->max_response_time ); - auto tracked_json = make_in_flight(std::move(json), my); - abstract_conn_ptr->send_response(std::move(*(*tracked_json)), code); + if( tracked_response->obj().has_value() ) { + std::string json = fc::json::to_string( *tracked_response->obj(), fc::time_point::now() + my->max_response_time ); + auto tracked_json = make_in_flight( std::move( json ), my ); + abstract_conn_ptr->send_response( std::move( tracked_json->obj() ), code ); + } else { + abstract_conn_ptr->send_response( {}, code ); + } } catch( ... ) { abstract_conn_ptr->handle_exception(); } @@ -584,7 +639,7 @@ class http_plugin_impl : public std::enable_shared_from_this { con->defer_http_response(); auto abstract_conn_ptr = make_abstract_conn_ptr(con, shared_from_this()); - if( !verify_max_bytes_in_flight( con )) return; + if( !verify_max_bytes_in_flight( con ) || !verify_max_requests_in_flight( con ) ) return; std::string resource = con->get_uri()->get_resource(); auto handler_itr = url_handlers.find( resource ); @@ -624,7 +679,7 @@ class http_plugin_impl : public std::enable_shared_from_this { } } - void add_aliases_for_endpoint( const tcp::endpoint& ep, string host, string port ) { + void add_aliases_for_endpoint( const tcp::endpoint& ep, const string& host, const string& port ) { auto resolved_port_str = std::to_string(ep.port()); valid_hosts.emplace(host + ":" + port); valid_hosts.emplace(host + ":" + resolved_port_str); @@ -641,7 +696,7 @@ class http_plugin_impl : public std::enable_shared_from_this { http_plugin::http_plugin():my(new http_plugin_impl()){ app().register_config_type(); } - http_plugin::~http_plugin(){} + http_plugin::~http_plugin() = default; void http_plugin::set_program_options(options_description&, options_description& cfg) { #ifdef BOOST_ASIO_HAS_LOCAL_SOCKETS @@ -652,7 +707,7 @@ class http_plugin_impl : public std::enable_shared_from_this { else cfg.add_options() ("unix-socket-path", bpo::value(), - "The filename (relative to data-dir) to create a unix socket for HTTP RPC; set blank to disable."); + "The filename (relative to data-dir) to create a unix socket for HTTP RPC; set blank to disable."); #endif if(current_http_plugin_defaults.default_http_port) @@ -709,7 +764,9 @@ class http_plugin_impl : public std::enable_shared_from_this { ("max-body-size", bpo::value()->default_value(1024*1024), "The maximum body size in bytes allowed for incoming RPC requests") ("http-max-bytes-in-flight-mb", bpo::value()->default_value(500), - "Maximum size in megabytes http_plugin should use for processing http requests. 503 error response when exceeded." ) + "Maximum size in megabytes http_plugin should use for processing http requests. 429 error response when exceeded." ) + ("http-max-in-flight-requests", bpo::value()->default_value(-1), + "Maximum number of requests http_plugin should use for processing http requests. 429 error response when exceeded." ) ("http-max-response-time-ms", bpo::value()->default_value(30), "Maximum time for processing a request.") ("verbose-http-errors", bpo::bool_switch()->default_value(false), @@ -736,9 +793,8 @@ class http_plugin_impl : public std::enable_shared_from_this { string lipstr = options.at( "http-server-address" ).as(); string host = lipstr.substr( 0, lipstr.find( ':' )); string port = lipstr.substr( host.size() + 1, lipstr.size()); - tcp::resolver::query query( tcp::v4(), host.c_str(), port.c_str()); try { - my->listen_endpoint = *resolver.resolve( query ); + my->listen_endpoint = *resolver.resolve( tcp::v4(), host, port ); ilog( "configured http to listen on ${h}:${p}", ("h", host)( "p", port )); } catch ( const boost::system::system_error& ec ) { elog( "failed to configure http to listen on ${h}:${p} (${m})", @@ -775,9 +831,8 @@ class http_plugin_impl : public std::enable_shared_from_this { string lipstr = options.at( "https-server-address" ).as(); string host = lipstr.substr( 0, lipstr.find( ':' )); string port = lipstr.substr( host.size() + 1, lipstr.size()); - tcp::resolver::query query( tcp::v4(), host.c_str(), port.c_str()); try { - my->https_listen_endpoint = *resolver.resolve( query ); + my->https_listen_endpoint = *resolver.resolve( tcp::v4(), host, port ); ilog( "configured https to listen on ${h}:${p} (TLS configuration will be validated momentarily)", ("h", host)( "p", port )); my->https_cert_chain = options.at( "https-certificate-chain-file" ).as(); @@ -801,6 +856,7 @@ class http_plugin_impl : public std::enable_shared_from_this { "http-threads ${num} must be greater than 0", ("num", my->thread_pool_size)); my->max_bytes_in_flight = options.at( "http-max-bytes-in-flight-mb" ).as() * 1024 * 1024; + my->max_requests_in_flight = options.at( "http-max-in-flight-requests" ).as(); my->max_response_time = fc::microseconds( options.at("http-max-response-time-ms").as() * 1000 ); //watch out for the returns above when adding new code here @@ -842,7 +898,7 @@ class http_plugin_impl : public std::enable_shared_from_this { my->unix_server.listen(*my->unix_endpoint); // captures `this`, my needs to live as long as unix_server is handling requests my->unix_server.set_http_handler([this](connection_hdl hdl) { - my->handle_http_request( my->unix_server.get_con_from_hdl(hdl)); + my->handle_http_request( my->unix_server.get_con_from_hdl(std::move(hdl))); }); my->unix_server.start_accept(); } catch ( const fc::exception& e ){ @@ -860,8 +916,8 @@ class http_plugin_impl : public std::enable_shared_from_this { if(my->https_listen_endpoint) { try { my->create_server_for_endpoint(*my->https_listen_endpoint, my->https_server); - my->https_server.set_tls_init_handler([this](websocketpp::connection_hdl hdl) -> ssl_context_ptr{ - return my->on_tls_init(hdl); + my->https_server.set_tls_init_handler([this](const websocketpp::connection_hdl& hdl) -> ssl_context_ptr{ + return my->on_tls_init(); }); fc_ilog( logger, "start listening for https requests" ); @@ -881,7 +937,7 @@ class http_plugin_impl : public std::enable_shared_from_this { add_api({{ std::string("/v1/node/get_supported_apis"), - [&](string, string body, url_response_callback cb) mutable { + [&](const string&, string body, url_response_callback cb) mutable { try { if (body.empty()) body = "{}"; auto result = (*this).get_supported_apis(); @@ -893,7 +949,7 @@ class http_plugin_impl : public std::enable_shared_from_this { }}); } catch (...) { fc_elog(logger, "http_plugin startup fails, shutting down"); - app().shutdown(); + app().quit(); } }); } @@ -957,13 +1013,8 @@ class http_plugin_impl : public std::enable_shared_from_this { } catch (fc::exception& e) { error_results results{500, "Internal Service Error", error_results::error_info(e, verbose_http_errors)}; cb( 500, fc::variant( results )); - if (e.code() != chain::greylist_net_usage_exceeded::code_value && - e.code() != chain::greylist_cpu_usage_exceeded::code_value && - e.code() != fc::timeout_exception::code_value ) { - fc_elog( logger, "FC Exception encountered while processing ${api}.${call}", - ("api", api_name)( "call", call_name ) ); - } - fc_dlog( logger, "Exception Details: ${e}", ("e", e.to_detail_string()) ); + fc_dlog( logger, "Exception while processing ${api}.${call}: ${e}", + ("api", api_name)( "call", call_name )("e", e.to_detail_string()) ); } catch (std::exception& e) { error_results results{500, "Internal Service Error", error_results::error_info(fc::exception( FC_LOG_MESSAGE( error, e.what())), verbose_http_errors)}; cb( 500, fc::variant( results )); diff --git a/plugins/http_plugin/include/eosio/http_plugin/http_plugin.hpp b/plugins/http_plugin/include/eosio/http_plugin/http_plugin.hpp index 79c002448b7..941ad1ceb34 100644 --- a/plugins/http_plugin/include/eosio/http_plugin/http_plugin.hpp +++ b/plugins/http_plugin/include/eosio/http_plugin/http_plugin.hpp @@ -1,8 +1,9 @@ #pragma once #include #include - #include +#include +#include namespace eosio { using namespace appbase; @@ -13,7 +14,7 @@ namespace eosio { * * Arguments: response_code, response_body */ - using url_response_callback = std::function; + using url_response_callback = std::function)>; /** * @brief Callback type for a URL handler @@ -64,13 +65,13 @@ namespace eosio { { public: http_plugin(); - virtual ~http_plugin(); + ~http_plugin() override; //must be called before initialize - static void set_defaults(const http_plugin_defaults config); + static void set_defaults(const http_plugin_defaults& config); APPBASE_PLUGIN_REQUIRES() - virtual void set_program_options(options_description&, options_description& cfg) override; + void set_program_options(options_description&, options_description& cfg) override; void plugin_initialize(const variables_map& options); void plugin_startup(); @@ -133,7 +134,7 @@ namespace eosio { static const uint8_t details_limit = 10; - error_info() {}; + error_info() = default; error_info(const fc::exception& exc, bool include_full_log) { code = exc.code(); @@ -157,6 +158,93 @@ namespace eosio { error_info error; }; + + /** + * @brief Used to trim whitespace from body. + * Returned string_view valid only for lifetime of body + */ + inline std::string_view make_trimmed_string_view(const std::string& body) { + if (body.empty()) { + return {}; + } + size_t left = 0; + size_t right = body.size() - 1; + + while(left < right) + { + if (body[left] == ' ') { + ++left; + } else { + break; + } + } + while(left < right) + { + if (body[right] == ' ') { + --right; + } else { + break; + } + } + if ((left == right) && (body[left] == ' ')) { + return {}; + } + return std::string_view(body).substr(left, right-left+1); + } + + inline bool is_empty_content(const std::string& body) { + const auto trimmed_body_view = make_trimmed_string_view(body); + if (trimmed_body_view.empty()) { + return true; + } + const size_t body_size = trimmed_body_view.size(); + if ((body_size > 1) && (trimmed_body_view.at(0) == '{') && (trimmed_body_view.at(body_size - 1) == '}')) { + for(size_t idx=1; idx + T parse_params(const std::string& body) { + if constexpr (params_type == http_params_types::params_required) { + if (is_empty_content(body)) { + EOS_THROW(chain::invalid_http_request, "A Request body is required"); + } + } + + try { + try { + if constexpr (params_type == http_params_types::no_params_required || params_type == http_params_types::possible_no_params) { + if (is_empty_content(body)) { + if constexpr (std::is_same_v) { + return std::string("{}"); + } + return {}; + } + if constexpr (params_type == http_params_types::no_params_required) { + EOS_THROW(chain::invalid_http_request, "no parameter should be given"); + } + } + return fc::json::from_string(body).as(); + } catch (const chain::chain_exception& e) { // EOS_RETHROW_EXCEPTIONS does not re-type these so, re-code it + throw fc::exception(e); + } + } EOS_RETHROW_EXCEPTIONS(chain::invalid_http_request, "Unable to parse valid input from POST body"); + } } FC_REFLECT(eosio::error_results::error_info::error_detail, (message)(file)(line_number)(method)) diff --git a/plugins/mongo_db_plugin/CMakeLists.txt b/plugins/mongo_db_plugin/CMakeLists.txt deleted file mode 100644 index 3fd67671616..00000000000 --- a/plugins/mongo_db_plugin/CMakeLists.txt +++ /dev/null @@ -1,66 +0,0 @@ -if(BUILD_MONGO_DB_PLUGIN) - - find_package(libmongoc-1.0 1.8) - - if (libmongoc-1.0_FOUND) - - find_package(libbson-1.0 REQUIRED) - message(STATUS "Found bson headers: ${BSON_INCLUDE_DIRS}") - - find_package(libbsoncxx-static REQUIRED) - find_package(libmongocxx-static REQUIRED) - find_package(libmongoc-static-1.0 REQUIRED) - find_package(libbson-static-1.0 REQUIRED) - - else() - message(FATAL_ERROR "Could NOT find mongo-c-driver. Disable mongo support or ensure mongo-c-driver and mongo-cxx-driver is built and installed") - return() - endif() - - # This needs to be after the else/return in the situation that libmongoc isn't found and we need to avoid building mongo :: 'bsoncxx/builder/basic/kvp.hpp' file not found - file(GLOB HEADERS "include/eosio/mongo_db_plugin/*.hpp") - add_library( mongo_db_plugin - mongo_db_plugin.cpp - ${HEADERS} ) - - target_include_directories(mongo_db_plugin - PRIVATE ${LIBMONGOCXX_STATIC_INCLUDE_DIRS} ${LIBBSONCXX_STATIC_INCLUDE_DIRS} ${BSON_INCLUDE_DIRS} - PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" - ) - - target_compile_definitions(mongo_db_plugin - PRIVATE ${LIBMONGOCXX_STATIC_DEFINITIONS} ${LIBBSONCXX_STATIC_DEFINITIONS} - ) - - # We can't just use *_STATIC_LIBRARIES variables to link against because the static - # variants of these may try to static link against libraries we don't want (like a system - # libc/c++). But we need to know if mongo c driver was built with ICU, SASL2, or snappy support - # so that we can continue to link to those. This certainly is a bit on the fragile side but - # try to parse what is included in MONGOC_STATIC_LIBRARIES to see what we should link to - foreach(MONGO_S_LIB ${MONGOC_STATIC_LIBRARIES}) - string(REGEX MATCH "libsasl2\\${CMAKE_SHARED_LIBRARY_SUFFIX}$" REGOUT ${MONGO_S_LIB}) - if(REGOUT) - set(LINK_SASL "sasl2") - endif() - - string(REGEX MATCH "libicuuc\\${CMAKE_SHARED_LIBRARY_SUFFIX}$" REGOUT ${MONGO_S_LIB}) - if(REGOUT) - set(LINK_ICU "icuuc") - endif() - - string(REGEX MATCH "libsnappy\\${CMAKE_SHARED_LIBRARY_SUFFIX}$" REGOUT ${MONGO_S_LIB}) - if(REGOUT) - set(LINK_SNAPPY "snappy") - endif() - endforeach() - - target_link_libraries(mongo_db_plugin - PUBLIC chain_plugin eosio_chain appbase - ${LIBMONGOCXX_STATIC_LIBRARY_PATH} ${LIBBSONCXX_STATIC_LIBRARY_PATH} - ${MONGOC_STATIC_LIBRARY} ${BSON_STATIC_LIBRARY} - resolv ${LINK_SASL} ${LINK_ICU} ${LINK_SNAPPY} - ) - -else() - message("mongo_db_plugin not selected and will be omitted.") -endif() diff --git a/plugins/mongo_db_plugin/include/eosio/mongo_db_plugin/bson.hpp b/plugins/mongo_db_plugin/include/eosio/mongo_db_plugin/bson.hpp deleted file mode 100644 index 984c5fd8b75..00000000000 --- a/plugins/mongo_db_plugin/include/eosio/mongo_db_plugin/bson.hpp +++ /dev/null @@ -1,246 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -namespace eosio { -void to_bson(const fc::variant_object& o, bsoncxx::builder::core& c); -void to_bson(const fc::variants& v, bsoncxx::builder::core& c); -void to_bson(const fc::variant& v, bsoncxx::builder::core& c); -bsoncxx::document::value to_bson(const fc::variant& v); - -void from_bson(const bsoncxx::document::view& view, fc::mutable_variant_object& o); -void from_bson(const bsoncxx::array::view& bson_array, fc::variants& a); -template void from_bson(const T& ele, fc::variant& v); -fc::variant from_bson(const bsoncxx::document::view& view); -} // namespace eosio - -namespace eosio { - -void to_bson(const fc::variant_object& o, bsoncxx::builder::core& c) -{ - auto itr = o.begin(); - while (itr != o.end()) { - c.key_owned(itr->key()); - to_bson(itr->value(), c); - ++itr; - } -} - -void to_bson(const fc::variants& a, bsoncxx::builder::core& c) -{ - auto itr = a.begin(); - while (itr != a.end()) { - to_bson(*itr, c); - ++itr; - } -} - -void to_bson(const fc::variant& v, bsoncxx::builder::core& c) -{ - switch (v.get_type()) { - case fc::variant::null_type: { - c.append(bsoncxx::types::b_null{}); - return; - } - case fc::variant::int64_type: - case fc::variant::uint64_type: { - c.append(v.as_int64()); - return; - } - case fc::variant::double_type: - c.append(v.as_double()); - return; - case fc::variant::bool_type: - c.append(v.as_bool()); - return; - case fc::variant::string_type: { - c.append(v.as_string()); - return; - } - case fc::variant::blob_type: { - bsoncxx::types::b_binary bin; - bin.sub_type = bsoncxx::binary_sub_type::k_binary; - bin.size = v.as_blob().data.size(); - bin.bytes = reinterpret_cast(&(*v.as_blob().data.begin())); - c.append(bin); - return; - } - case fc::variant::array_type: { - const fc::variants& a = v.get_array(); - bsoncxx::builder::core sub(true); - to_bson(a, sub); - c.append(sub.extract_array()); - return; - } - case fc::variant::object_type: { - const fc::variant_object& o = v.get_object(); - if (o.size() == 1) { - const auto value = o.begin()->value(); - if (o.begin()->key().compare("$oid") == 0) { - if (value.get_type() == fc::variant::string_type - && bson_oid_is_valid(value.as_string().data(), value.as_string().size())) { - bsoncxx::oid oid(value.as_string()); - c.append(oid); - break; - } - } - else if (o.begin()->key().compare("$date") == 0) { - if (value.get_type() == fc::variant::int64_type) { - bsoncxx::types::b_date date(std::chrono::milliseconds(value.as_int64())); - c.append(date); - break; - } - else if (value.get_type() == fc::variant::object_type) { - const fc::variant_object& obj = value.get_object(); - if (obj.size() == 1) { - auto number = obj.begin(); - if (number->key().compare("$numberLong") == 0) { - bsoncxx::types::b_date date(std::chrono::milliseconds(number->value().as_int64())); - c.append(date); - break; - } - } - } - } - else if (o.begin()->key().compare("$timestamp") == 0) { - if (value.get_type() == fc::variant::object_type) { - const fc::variant_object& obj = value.get_object(); - if (obj.size() == 2) { - auto t = obj.begin(); - auto i = t; - ++i; - if (t->key().compare("t") == 0 && i->key().compare("i") == 0) { - bsoncxx::types::b_timestamp ts; - ts.timestamp = static_cast(t->value().as_uint64()); - ts.increment = static_cast(i->value().as_uint64()); - c.append(ts); - break; - } - } - } - } - } - bsoncxx::builder::core sub(false); - to_bson(o, sub); - c.append(sub.extract_document()); - return; - } - default: - FC_THROW_EXCEPTION( - fc::invalid_arg_exception, - "Unsupported fc::variant type: " + std::to_string(v.get_type())); - } -} - -bsoncxx::document::value to_bson(const fc::variant& v) -{ - bsoncxx::builder::core doc(false); - if (v.get_type() == fc::variant::object_type) { - const fc::variant_object& o = v.get_object(); - to_bson(o, doc); - } - else if (v.get_type() != fc::variant::null_type) { - FC_THROW_EXCEPTION( - fc::invalid_arg_exception, - "Unsupported root fc::variant type: " + std::to_string(v.get_type())); - } - return doc.extract_document(); -} - -void from_bson(const bsoncxx::document::view& view, fc::mutable_variant_object& o) -{ - for (bsoncxx::document::element ele : view) { - fc::variant v; - from_bson(ele, v); - o(ele.key().data(), v); - } -} - -void from_bson(const bsoncxx::array::view& bson_array, fc::variants& a) -{ - a.reserve(std::distance(bson_array.cbegin(), bson_array.cend())); - for (bsoncxx::array::element ele : bson_array) { - fc::variant v; - from_bson(ele, v); - a.push_back(v); - } -} - -template -void from_bson(const T& ele, fc::variant& v) -{ - switch (ele.type()) { - case bsoncxx::type::k_double: - v = ele.get_double().value; - return; - case bsoncxx::type::k_utf8: - v = bsoncxx::string::to_string(ele.get_utf8().value); - return; - case bsoncxx::type::k_document: { - fc::mutable_variant_object o; - from_bson(ele.get_document().value, o); - v = o; - return; - } - case bsoncxx::type::k_array: { - bsoncxx::array::view sub_array{ele.get_array().value}; - fc::variants a; - from_bson(sub_array, a); - v = a; - return; - } - case bsoncxx::type::k_binary: { - fc::blob blob; - blob.data.resize(ele.get_binary().size); - std::copy(ele.get_binary().bytes, ele.get_binary().bytes + ele.get_binary().size, blob.data.begin()); - v = blob; - return; - } - case bsoncxx::type::k_undefined: - case bsoncxx::type::k_null: - v = fc::variant(); - return; - case bsoncxx::type::k_oid: - v = fc::variant_object("$oid", ele.get_oid().value.to_string()); - return; - case bsoncxx::type::k_bool: - v = ele.get_bool().value; - return; - case bsoncxx::type::k_date: - v = fc::variant_object("$date", ele.get_date().to_int64()); - return; - case bsoncxx::type::k_int32: - v = ele.get_int32().value; - return; - case bsoncxx::type::k_timestamp: - v = fc::variant_object("$timestamp", fc::mutable_variant_object("t", ele.get_timestamp().timestamp)("i", ele.get_timestamp().increment)); - return; - case bsoncxx::type::k_int64: - v = ele.get_int64().value; - return; - default: - FC_THROW_EXCEPTION( - fc::invalid_arg_exception, - "Unsupported BSON type: " + bsoncxx::to_string(ele.type())); - } -} - -fc::variant from_bson(const bsoncxx::document::view& view) -{ - fc::mutable_variant_object o; - from_bson(view, o); - return o; -} - -} // namespace eosio - diff --git a/plugins/mongo_db_plugin/include/eosio/mongo_db_plugin/mongo_db_plugin.hpp b/plugins/mongo_db_plugin/include/eosio/mongo_db_plugin/mongo_db_plugin.hpp deleted file mode 100644 index 8dfbf09bff1..00000000000 --- a/plugins/mongo_db_plugin/include/eosio/mongo_db_plugin/mongo_db_plugin.hpp +++ /dev/null @@ -1,44 +0,0 @@ -#pragma once - -#include -#include -#include - -namespace eosio { - -using mongo_db_plugin_impl_ptr = std::shared_ptr; - -/** - * Provides persistence to MongoDB for: - * accounts - * actions - * block_states - * blocks - * transaction_traces - * transactions - * pub_keys - * account_controls - * - * See data dictionary (DB Schema Definition - EOS API) for description of MongoDB schema. - * - * If cmake -DBUILD_MONGO_DB_PLUGIN=true not specified then this plugin not compiled/included. - */ -class mongo_db_plugin : public plugin { -public: - APPBASE_PLUGIN_REQUIRES((chain_plugin)) - - mongo_db_plugin(); - virtual ~mongo_db_plugin(); - - virtual void set_program_options(options_description& cli, options_description& cfg) override; - - void plugin_initialize(const variables_map& options); - void plugin_startup(); - void plugin_shutdown(); - -private: - mongo_db_plugin_impl_ptr my; -}; - -} - diff --git a/plugins/mongo_db_plugin/mongo_db_plugin.cpp b/plugins/mongo_db_plugin/mongo_db_plugin.cpp deleted file mode 100644 index 541e522f593..00000000000 --- a/plugins/mongo_db_plugin/mongo_db_plugin.cpp +++ /dev/null @@ -1,1691 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include - -#include - -#include -#include -#include -#include -#include - -namespace fc { class variant; } - -namespace eosio { - -using chain::account_name; -using chain::action_name; -using chain::block_id_type; -using chain::permission_name; -using chain::transaction; -using chain::signed_transaction; -using chain::signed_block; -using chain::transaction_id_type; -using chain::packed_transaction; - -static appbase::abstract_plugin& _mongo_db_plugin = app().register_plugin(); - -struct filter_entry { - name receiver; - name action; - name actor; - - friend bool operator<( const filter_entry& a, const filter_entry& b ) { - return std::tie( a.receiver, a.action, a.actor ) < std::tie( b.receiver, b.action, b.actor ); - } - - // receiver action actor - bool match( const name& rr, const name& an, const name& ar ) const { - return (receiver.to_uint64_t() == 0 || receiver == rr) && - (action.to_uint64_t() == 0 || action == an) && - (actor.to_uint64_t() == 0 || actor == ar); - } -}; - -class mongo_db_plugin_impl { -public: - mongo_db_plugin_impl(); - ~mongo_db_plugin_impl(); - - fc::optional accepted_block_connection; - fc::optional irreversible_block_connection; - fc::optional accepted_transaction_connection; - fc::optional applied_transaction_connection; - - void consume_blocks(); - - void accepted_block( const chain::block_state_ptr& ); - void applied_irreversible_block(const chain::block_state_ptr&); - void accepted_transaction(const chain::transaction_metadata_ptr&); - void applied_transaction(const chain::transaction_trace_ptr&); - void process_accepted_transaction(const chain::transaction_metadata_ptr&); - void _process_accepted_transaction(const chain::transaction_metadata_ptr&); - void process_applied_transaction(const chain::transaction_trace_ptr&); - void _process_applied_transaction(const chain::transaction_trace_ptr&); - void process_accepted_block( const chain::block_state_ptr& ); - void _process_accepted_block( const chain::block_state_ptr& ); - void process_irreversible_block(const chain::block_state_ptr&); - void _process_irreversible_block(const chain::block_state_ptr&); - - optional get_abi_serializer( account_name n ); - template fc::variant to_variant_with_abi( const T& obj ); - - void purge_abi_cache(); - - bool add_action_trace( mongocxx::bulk_write& bulk_action_traces, const chain::action_trace& atrace, - const chain::transaction_trace_ptr& t, - bool executed, const std::chrono::milliseconds& now, - bool& write_ttrace ); - - void update_account(const chain::action& act); - - void add_pub_keys( const vector& keys, const account_name& name, - const permission_name& permission, const std::chrono::milliseconds& now ); - void remove_pub_keys( const account_name& name, const permission_name& permission ); - void add_account_control( const vector& controlling_accounts, - const account_name& name, const permission_name& permission, - const std::chrono::milliseconds& now ); - void remove_account_control( const account_name& name, const permission_name& permission ); - - /// @return true if act should be added to mongodb, false to skip it - bool filter_include( const account_name& receiver, const action_name& act_name, - const vector& authorization ) const; - bool filter_include( const transaction& trx ) const; - - void init(); - void wipe_database(); - void create_expiration_index(mongocxx::collection& collection, uint32_t expire_after_seconds); - - template void queue(Queue& queue, const Entry& e); - - bool configured{false}; - bool wipe_database_on_startup{false}; - uint32_t start_block_num = 0; - std::atomic_bool start_block_reached{false}; - - bool is_producer = false; - bool filter_on_star = true; - std::set filter_on; - std::set filter_out; - bool update_blocks_via_block_num = false; - bool store_blocks = true; - bool store_block_states = true; - bool store_transactions = true; - bool store_transaction_traces = true; - bool store_action_traces = true; - uint32_t expire_after_seconds = 0; - - std::string db_name; - mongocxx::instance mongo_inst; - fc::optional mongo_pool; - - // consum thread - mongocxx::collection _accounts; - mongocxx::collection _trans; - mongocxx::collection _trans_traces; - mongocxx::collection _action_traces; - mongocxx::collection _block_states; - mongocxx::collection _blocks; - mongocxx::collection _pub_keys; - mongocxx::collection _account_controls; - - size_t max_queue_size = 0; - int queue_sleep_time = 0; - size_t abi_cache_size = 0; - std::deque transaction_metadata_queue; - std::deque transaction_metadata_process_queue; - std::deque transaction_trace_queue; - std::deque transaction_trace_process_queue; - std::deque block_state_queue; - std::deque block_state_process_queue; - std::deque irreversible_block_state_queue; - std::deque irreversible_block_state_process_queue; - std::mutex mtx; - std::condition_variable condition; - std::thread consume_thread; - std::atomic_bool done{false}; - std::atomic_bool startup{true}; - fc::optional chain_id; - fc::microseconds abi_serializer_max_time; - - struct by_account; - struct by_last_access; - - struct abi_cache { - account_name account; - fc::time_point last_accessed; - fc::optional serializer; - }; - - typedef boost::multi_index_container, member >, - ordered_non_unique< tag, member > - > - > abi_cache_index_t; - - abi_cache_index_t abi_cache_index; - - static const action_name newaccount; - static const action_name setabi; - static const action_name updateauth; - static const action_name deleteauth; - static const permission_name owner; - static const permission_name active; - - static const std::string block_states_col; - static const std::string blocks_col; - static const std::string trans_col; - static const std::string trans_traces_col; - static const std::string action_traces_col; - static const std::string accounts_col; - static const std::string pub_keys_col; - static const std::string account_controls_col; -}; - -const action_name mongo_db_plugin_impl::newaccount = chain::newaccount::get_name(); -const action_name mongo_db_plugin_impl::setabi = chain::setabi::get_name(); -const action_name mongo_db_plugin_impl::updateauth = chain::updateauth::get_name(); -const action_name mongo_db_plugin_impl::deleteauth = chain::deleteauth::get_name(); -const permission_name mongo_db_plugin_impl::owner = chain::config::owner_name; -const permission_name mongo_db_plugin_impl::active = chain::config::active_name; - -const std::string mongo_db_plugin_impl::block_states_col = "block_states"; -const std::string mongo_db_plugin_impl::blocks_col = "blocks"; -const std::string mongo_db_plugin_impl::trans_col = "transactions"; -const std::string mongo_db_plugin_impl::trans_traces_col = "transaction_traces"; -const std::string mongo_db_plugin_impl::action_traces_col = "action_traces"; -const std::string mongo_db_plugin_impl::accounts_col = "accounts"; -const std::string mongo_db_plugin_impl::pub_keys_col = "pub_keys"; -const std::string mongo_db_plugin_impl::account_controls_col = "account_controls"; - -bool mongo_db_plugin_impl::filter_include( const account_name& receiver, const action_name& act_name, - const vector& authorization ) const -{ - bool include = false; - if( filter_on_star ) { - include = true; - } else { - auto itr = std::find_if( filter_on.cbegin(), filter_on.cend(), [&receiver, &act_name]( const auto& filter ) { - return filter.match( receiver, act_name, {} ); - } ); - if( itr != filter_on.cend() ) { - include = true; - } else { - for( const auto& a : authorization ) { - auto itr = std::find_if( filter_on.cbegin(), filter_on.cend(), [&receiver, &act_name, &a]( const auto& filter ) { - return filter.match( receiver, act_name, a.actor ); - } ); - if( itr != filter_on.cend() ) { - include = true; - break; - } - } - } - } - - if( !include ) { return false; } - if( filter_out.empty() ) { return true; } - - auto itr = std::find_if( filter_out.cbegin(), filter_out.cend(), [&receiver, &act_name]( const auto& filter ) { - return filter.match( receiver, act_name, {} ); - } ); - if( itr != filter_out.cend() ) { return false; } - - for( const auto& a : authorization ) { - auto itr = std::find_if( filter_out.cbegin(), filter_out.cend(), [&receiver, &act_name, &a]( const auto& filter ) { - return filter.match( receiver, act_name, a.actor ); - } ); - if( itr != filter_out.cend() ) { return false; } - } - - return true; -} - -bool mongo_db_plugin_impl::filter_include( const transaction& trx ) const -{ - if( !filter_on_star || !filter_out.empty() ) { - bool include = false; - for( const auto& a : trx.actions ) { - if( filter_include( a.account, a.name, a.authorization ) ) { - include = true; - break; - } - } - if( !include ) { - for( const auto& a : trx.context_free_actions ) { - if( filter_include( a.account, a.name, a.authorization ) ) { - include = true; - break; - } - } - } - return include; - } - return true; -} - - -template -void mongo_db_plugin_impl::queue( Queue& queue, const Entry& e ) { - std::unique_lock lock( mtx ); - auto queue_size = queue.size(); - if( queue_size > max_queue_size ) { - lock.unlock(); - condition.notify_one(); - queue_sleep_time += 10; - if( queue_sleep_time > 1000 ) - wlog("queue size: ${q}", ("q", queue_size)); - std::this_thread::sleep_for( std::chrono::milliseconds( queue_sleep_time )); - lock.lock(); - } else { - queue_sleep_time -= 10; - if( queue_sleep_time < 0 ) queue_sleep_time = 0; - } - queue.emplace_back( e ); - lock.unlock(); - condition.notify_one(); -} - -void mongo_db_plugin_impl::accepted_transaction( const chain::transaction_metadata_ptr& t ) { - try { - if( store_transactions ) { - queue( transaction_metadata_queue, t ); - } - } catch (fc::exception& e) { - elog("FC Exception while accepted_transaction ${e}", ("e", e.to_string())); - } catch (std::exception& e) { - elog("STD Exception while accepted_transaction ${e}", ("e", e.what())); - } catch (...) { - elog("Unknown exception while accepted_transaction"); - } -} - -void mongo_db_plugin_impl::applied_transaction( const chain::transaction_trace_ptr& t ) { - try { - // Traces emitted from an incomplete block leave the producer_block_id as empty. - // - // Avoid adding the action traces or transaction traces to the database if the producer_block_id is empty. - // This way traces from speculatively executed transactions are not included in the Mongo database which can - // avoid potential confusion for consumers of that database. - // - // Due to forks, it could be possible for multiple incompatible action traces with the same block_num and trx_id - // to exist in the database. And if the producer double produces a block, even the block_time may not - // disambiguate the two action traces. Without a producer_block_id to disambiguate and determine if the action - // trace comes from an orphaned fork branching off of the blockchain, consumers of the Mongo DB database may be - // reacting to a stale action trace that never actually executed in the current blockchain. - // - // It is better to avoid this potential confusion by not logging traces from speculative execution, i.e. emitted - // from an incomplete block. This means that traces will not be recorded in speculative read-mode, but - // users should not be using the mongo_db_plugin in that mode anyway. - // - // Allow logging traces if node is a producer for testing purposes, so a single nodeos can do both for testing. - // - // It is recommended to run mongo_db_plugin in read-mode = read-only. - // - if( !is_producer && !t->producer_block_id.valid() ) - return; - // always queue since account information always gathered - queue( transaction_trace_queue, t ); - } catch (fc::exception& e) { - elog("FC Exception while applied_transaction ${e}", ("e", e.to_string())); - } catch (std::exception& e) { - elog("STD Exception while applied_transaction ${e}", ("e", e.what())); - } catch (...) { - elog("Unknown exception while applied_transaction"); - } -} - -void mongo_db_plugin_impl::applied_irreversible_block( const chain::block_state_ptr& bs ) { - try { - if( store_blocks || store_block_states || store_transactions ) { - queue( irreversible_block_state_queue, bs ); - } - } catch (fc::exception& e) { - elog("FC Exception while applied_irreversible_block ${e}", ("e", e.to_string())); - } catch (std::exception& e) { - elog("STD Exception while applied_irreversible_block ${e}", ("e", e.what())); - } catch (...) { - elog("Unknown exception while applied_irreversible_block"); - } -} - -void mongo_db_plugin_impl::accepted_block( const chain::block_state_ptr& bs ) { - try { - if( !start_block_reached ) { - if( bs->block_num >= start_block_num ) { - start_block_reached = true; - } - } - if( store_blocks || store_block_states ) { - queue( block_state_queue, bs ); - } - } catch (fc::exception& e) { - elog("FC Exception while accepted_block ${e}", ("e", e.to_string())); - } catch (std::exception& e) { - elog("STD Exception while accepted_block ${e}", ("e", e.what())); - } catch (...) { - elog("Unknown exception while accepted_block"); - } -} - -void mongo_db_plugin_impl::consume_blocks() { - try { - auto mongo_client = mongo_pool->acquire(); - auto& mongo_conn = *mongo_client; - - _accounts = mongo_conn[db_name][accounts_col]; - _trans = mongo_conn[db_name][trans_col]; - _trans_traces = mongo_conn[db_name][trans_traces_col]; - _action_traces = mongo_conn[db_name][action_traces_col]; - _blocks = mongo_conn[db_name][blocks_col]; - _block_states = mongo_conn[db_name][block_states_col]; - _pub_keys = mongo_conn[db_name][pub_keys_col]; - _account_controls = mongo_conn[db_name][account_controls_col]; - - while (true) { - std::unique_lock lock(mtx); - while ( transaction_metadata_queue.empty() && - transaction_trace_queue.empty() && - block_state_queue.empty() && - irreversible_block_state_queue.empty() && - !done ) { - condition.wait(lock); - } - - // capture for processing - size_t transaction_metadata_size = transaction_metadata_queue.size(); - if (transaction_metadata_size > 0) { - transaction_metadata_process_queue = move(transaction_metadata_queue); - transaction_metadata_queue.clear(); - } - size_t transaction_trace_size = transaction_trace_queue.size(); - if (transaction_trace_size > 0) { - transaction_trace_process_queue = move(transaction_trace_queue); - transaction_trace_queue.clear(); - } - size_t block_state_size = block_state_queue.size(); - if (block_state_size > 0) { - block_state_process_queue = move(block_state_queue); - block_state_queue.clear(); - } - size_t irreversible_block_size = irreversible_block_state_queue.size(); - if (irreversible_block_size > 0) { - irreversible_block_state_process_queue = move(irreversible_block_state_queue); - irreversible_block_state_queue.clear(); - } - - lock.unlock(); - - if (done) { - ilog("draining queue, size: ${q}", ("q", transaction_metadata_size + transaction_trace_size + block_state_size + irreversible_block_size)); - } - - // process transactions - auto start_time = fc::time_point::now(); - auto size = transaction_trace_process_queue.size(); - while (!transaction_trace_process_queue.empty()) { - const auto& t = transaction_trace_process_queue.front(); - process_applied_transaction(t); - transaction_trace_process_queue.pop_front(); - } - auto time = fc::time_point::now() - start_time; - auto per = size > 0 ? time.count()/size : 0; - if( time > fc::microseconds(500000) ) // reduce logging, .5 secs - ilog( "process_applied_transaction, time per: ${p}, size: ${s}, time: ${t}", ("s", size)("t", time)("p", per) ); - - start_time = fc::time_point::now(); - size = transaction_metadata_process_queue.size(); - while (!transaction_metadata_process_queue.empty()) { - const auto& t = transaction_metadata_process_queue.front(); - process_accepted_transaction(t); - transaction_metadata_process_queue.pop_front(); - } - time = fc::time_point::now() - start_time; - per = size > 0 ? time.count()/size : 0; - if( time > fc::microseconds(500000) ) // reduce logging, .5 secs - ilog( "process_accepted_transaction, time per: ${p}, size: ${s}, time: ${t}", ("s", size)( "t", time )( "p", per )); - - // process blocks - start_time = fc::time_point::now(); - size = block_state_process_queue.size(); - while (!block_state_process_queue.empty()) { - const auto& bs = block_state_process_queue.front(); - process_accepted_block( bs ); - block_state_process_queue.pop_front(); - } - time = fc::time_point::now() - start_time; - per = size > 0 ? time.count()/size : 0; - if( time > fc::microseconds(500000) ) // reduce logging, .5 secs - ilog( "process_accepted_block, time per: ${p}, size: ${s}, time: ${t}", ("s", size)("t", time)("p", per) ); - - // process irreversible blocks - start_time = fc::time_point::now(); - size = irreversible_block_state_process_queue.size(); - while (!irreversible_block_state_process_queue.empty()) { - const auto& bs = irreversible_block_state_process_queue.front(); - process_irreversible_block(bs); - irreversible_block_state_process_queue.pop_front(); - } - time = fc::time_point::now() - start_time; - per = size > 0 ? time.count()/size : 0; - if( time > fc::microseconds(500000) ) // reduce logging, .5 secs - ilog( "process_irreversible_block, time per: ${p}, size: ${s}, time: ${t}", ("s", size)("t", time)("p", per) ); - - if( transaction_metadata_size == 0 && - transaction_trace_size == 0 && - block_state_size == 0 && - irreversible_block_size == 0 && - done ) { - break; - } - } - ilog("mongo_db_plugin consume thread shutdown gracefully"); - } catch (fc::exception& e) { - elog("FC Exception while consuming block ${e}", ("e", e.to_string())); - } catch (std::exception& e) { - elog("STD Exception while consuming block ${e}", ("e", e.what())); - } catch (...) { - elog("Unknown exception while consuming block"); - } -} - -namespace { - -auto find_account( mongocxx::collection& accounts, const account_name& name ) { - using bsoncxx::builder::basic::make_document; - using bsoncxx::builder::basic::kvp; - return accounts.find_one( make_document( kvp( "name", name.to_string()))); -} - -auto find_block( mongocxx::collection& blocks, const string& id ) { - using bsoncxx::builder::basic::make_document; - using bsoncxx::builder::basic::kvp; - - mongocxx::options::find options; - options.projection( make_document( kvp( "_id", 1 )) ); // only return _id - return blocks.find_one( make_document( kvp( "block_id", id )), options); -} - -void handle_mongo_exception( const std::string& desc, int line_num ) { - bool shutdown = true; - try { - try { - throw; - } catch( mongocxx::logic_error& e) { - // logic_error on invalid key, do not shutdown - wlog( "mongo logic error, ${desc}, line ${line}, code ${code}, ${what}", - ("desc", desc)( "line", line_num )( "code", e.code().value() )( "what", e.what() )); - shutdown = false; - } catch( mongocxx::operation_exception& e) { - elog( "mongo exception, ${desc}, line ${line}, code ${code}, ${details}", - ("desc", desc)( "line", line_num )( "code", e.code().value() )( "details", e.code().message() )); - if (e.raw_server_error()) { - elog( " raw_server_error: ${e}", ( "e", bsoncxx::to_json(e.raw_server_error()->view()))); - } - } catch( mongocxx::exception& e) { - elog( "mongo exception, ${desc}, line ${line}, code ${code}, ${what}", - ("desc", desc)( "line", line_num )( "code", e.code().value() )( "what", e.what() )); - } catch( bsoncxx::exception& e) { - elog( "bsoncxx exception, ${desc}, line ${line}, code ${code}, ${what}", - ("desc", desc)( "line", line_num )( "code", e.code().value() )( "what", e.what() )); - } catch( fc::exception& er ) { - elog( "mongo fc exception, ${desc}, line ${line}, ${details}", - ("desc", desc)( "line", line_num )( "details", er.to_detail_string())); - } catch( const std::exception& e ) { - elog( "mongo std exception, ${desc}, line ${line}, ${what}", - ("desc", desc)( "line", line_num )( "what", e.what())); - } catch( ... ) { - elog( "mongo unknown exception, ${desc}, line ${line_nun}", ("desc", desc)( "line_num", line_num )); - } - } catch (...) { - std::cerr << "Exception attempting to handle exception for " << desc << " " << line_num << std::endl; - } - - if( shutdown ) { - // shutdown if mongo failed to provide opportunity to fix issue and restart - app().quit(); - } -} - -// custom oid to avoid monotonic throttling -// https://docs.mongodb.com/master/core/bulk-write-operations/#avoid-monotonic-throttling -bsoncxx::oid make_custom_oid() { - bsoncxx::oid x = bsoncxx::oid(); - const char* p = x.bytes(); - std::swap((short&)p[0], (short&)p[10]); - return x; -} - -} // anonymous namespace - -void mongo_db_plugin_impl::purge_abi_cache() { - if( abi_cache_index.size() < abi_cache_size ) return; - - // remove the oldest (smallest) last accessed - auto& idx = abi_cache_index.get(); - auto itr = idx.begin(); - if( itr != idx.end() ) { - idx.erase( itr ); - } -} - -optional mongo_db_plugin_impl::get_abi_serializer( account_name n ) { - using bsoncxx::builder::basic::kvp; - using bsoncxx::builder::basic::make_document; - if( n.good()) { - try { - - auto itr = abi_cache_index.find( n ); - if( itr != abi_cache_index.end() ) { - abi_cache_index.modify( itr, []( auto& entry ) { - entry.last_accessed = fc::time_point::now(); - }); - - return itr->serializer; - } - - auto account = _accounts.find_one( make_document( kvp("name", n.to_string())) ); - if(account) { - auto view = account->view(); - abi_def abi; - if( view.find( "abi" ) != view.end()) { - try { - abi = from_bson( view["abi"].get_document() ).as(); - } catch (...) { - ilog( "Unable to convert account abi to abi_def for ${n}", ( "n", n )); - return optional(); - } - - purge_abi_cache(); // make room if necessary - abi_cache entry; - entry.account = n; - entry.last_accessed = fc::time_point::now(); - abi_serializer abis; - if( n == chain::config::system_account_name ) { - // redefine eosio setabi.abi from bytes to abi_def - // Done so that abi is stored as abi_def in mongo instead of as bytes - auto itr = std::find_if( abi.structs.begin(), abi.structs.end(), - []( const auto& s ) { return s.name == "setabi"; } ); - if( itr != abi.structs.end() ) { - auto itr2 = std::find_if( itr->fields.begin(), itr->fields.end(), - []( const auto& f ) { return f.name == "abi"; } ); - if( itr2 != itr->fields.end() ) { - if( itr2->type == "bytes" ) { - itr2->type = "abi_def"; - // unpack setabi.abi as abi_def instead of as bytes - abis.add_specialized_unpack_pack( "abi_def", - std::make_pair( - []( fc::datastream& stream, bool is_array, bool is_optional, const abi_serializer::yield_function_t& yield ) { - EOS_ASSERT( !is_array && !is_optional, chain::mongo_db_exception, "unexpected abi_def"); - chain::bytes temp; - fc::raw::unpack( stream, temp ); - return fc::variant( fc::raw::unpack( temp ) ); - }, - []( const fc::variant& var, fc::datastream& ds, bool is_array, bool is_optional, const abi_serializer::yield_function_t& yield ) { - EOS_ASSERT( false, chain::mongo_db_exception, "never called" ); - } - ) ); - } - } - } - } - // mongo does not like empty json keys - // make abi_serializer use empty_name instead of "" for the action data - for( auto& s : abi.structs ) { - if( s.name.empty() ) { - s.name = "empty_struct_name"; - } - for( auto& f : s.fields ) { - if( f.name.empty() ) { - f.name = "empty_field_name"; - } - } - } - abis.set_abi( abi, abi_serializer::create_yield_function( abi_serializer_max_time ) ); - entry.serializer.emplace( std::move( abis ) ); - abi_cache_index.insert( entry ); - return entry.serializer; - } - } - } FC_CAPTURE_AND_LOG((n)) - } - return optional(); -} - -template -fc::variant mongo_db_plugin_impl::to_variant_with_abi( const T& obj ) { - fc::variant pretty_output; - abi_serializer::to_variant( obj, pretty_output, - [&]( account_name n ) { return get_abi_serializer( n ); }, - abi_serializer::create_yield_function( abi_serializer_max_time ) ); - return pretty_output; -} - -void mongo_db_plugin_impl::process_accepted_transaction( const chain::transaction_metadata_ptr& t ) { - try { - if( start_block_reached ) { - _process_accepted_transaction( t ); - } - } catch (fc::exception& e) { - elog("FC Exception while processing accepted transaction metadata: ${e}", ("e", e.to_detail_string())); - } catch (std::exception& e) { - elog("STD Exception while processing accepted tranasction metadata: ${e}", ("e", e.what())); - } catch (...) { - elog("Unknown exception while processing accepted transaction metadata"); - } -} - -void mongo_db_plugin_impl::process_applied_transaction( const chain::transaction_trace_ptr& t ) { - try { - // always call since we need to capture setabi on accounts even if not storing transaction traces - _process_applied_transaction( t ); - } catch (fc::exception& e) { - elog("FC Exception while processing applied transaction trace: ${e}", ("e", e.to_detail_string())); - } catch (std::exception& e) { - elog("STD Exception while processing applied transaction trace: ${e}", ("e", e.what())); - } catch (...) { - elog("Unknown exception while processing applied transaction trace"); - } -} - -void mongo_db_plugin_impl::process_irreversible_block(const chain::block_state_ptr& bs) { - try { - if( start_block_reached ) { - _process_irreversible_block( bs ); - } - } catch (fc::exception& e) { - elog("FC Exception while processing irreversible block: ${e}", ("e", e.to_detail_string())); - } catch (std::exception& e) { - elog("STD Exception while processing irreversible block: ${e}", ("e", e.what())); - } catch (...) { - elog("Unknown exception while processing irreversible block"); - } -} - -void mongo_db_plugin_impl::process_accepted_block( const chain::block_state_ptr& bs ) { - try { - if( start_block_reached ) { - _process_accepted_block( bs ); - } - } catch (fc::exception& e) { - elog("FC Exception while processing accepted block trace ${e}", ("e", e.to_string())); - } catch (std::exception& e) { - elog("STD Exception while processing accepted block trace ${e}", ("e", e.what())); - } catch (...) { - elog("Unknown exception while processing accepted block trace"); - } -} - -void mongo_db_plugin_impl::_process_accepted_transaction( const chain::transaction_metadata_ptr& t ) { - using namespace bsoncxx::types; - using bsoncxx::builder::basic::kvp; - using bsoncxx::builder::basic::make_document; - using bsoncxx::builder::basic::make_array; - namespace bbb = bsoncxx::builder::basic; - - const signed_transaction& trx = t->packed_trx()->get_signed_transaction(); - - if( !filter_include( trx ) ) return; - - auto trans_doc = bsoncxx::builder::basic::document{}; - - auto now = std::chrono::duration_cast( - std::chrono::microseconds{fc::time_point::now().time_since_epoch().count()} ); - - const auto& trx_id = t->id(); - const auto trx_id_str = trx_id.str(); - - trans_doc.append( kvp( "trx_id", trx_id_str ) ); - - auto v = to_variant_with_abi( trx ); - - try { - const auto& trx_value = to_bson( v ); - trans_doc.append( bsoncxx::builder::concatenate_doc{trx_value.view()} ); - } catch( bsoncxx::exception& e) { - elog( "Unable to convert transaction to BSON: ${e}", ("e", e.what()) ); - try { - elog( " JSON: ${j}", ("j", fc::json::to_string( v, fc::time_point::now() + fc::exception::format_time_limit )) ); - } catch(...) {} - } - - fc::variant signing_keys; - const flat_set& keys = t->recovered_keys(); - if( !keys.empty() ) { - signing_keys = keys; - } else { - flat_set pub_keys; - trx.get_signature_keys( *chain_id, fc::time_point::maximum(), pub_keys, false ); - if( !pub_keys.empty() ) { - signing_keys = pub_keys; - } - } - - if( signing_keys.get_type() == fc::variant::array_type && signing_keys.get_array().size() > 0) { - try { - bsoncxx::builder::core keys_value(true); - to_bson( signing_keys.get_array(), keys_value ); - trans_doc.append( kvp( "signing_keys", keys_value.extract_array() ) ); - } catch( bsoncxx::exception& e ) { - elog( "Unable to convert signing keys to BSON: ${e}", ("e", e.what()) ); - try { - elog( " JSON: ${j}", ("j", fc::json::to_string( signing_keys, fc::time_point::now() + fc::exception::format_time_limit )) ); - } catch(...) {} - } - } - - trans_doc.append( kvp( "accepted", b_bool{t->accepted} ) ); - trans_doc.append( kvp( "implicit", b_bool{t->implicit} ) ); - trans_doc.append( kvp( "scheduled", b_bool{t->scheduled} ) ); - - trans_doc.append( kvp( "createdAt", b_date{now} ) ); - - try { - mongocxx::options::update update_opts{}; - update_opts.upsert( true ); - if( !_trans.update_one( make_document( kvp( "trx_id", trx_id_str ) ), - make_document( kvp( "$set", trans_doc.view() ) ), update_opts ) ) { - EOS_ASSERT( false, chain::mongo_db_insert_fail, "Failed to insert trans ${id}", ("id", trx_id) ); - } - } catch( ... ) { - handle_mongo_exception( "trans insert", __LINE__ ); - } - -} - -bool -mongo_db_plugin_impl::add_action_trace( mongocxx::bulk_write& bulk_action_traces, const chain::action_trace& atrace, - const chain::transaction_trace_ptr& t, - bool executed, const std::chrono::milliseconds& now, - bool& write_ttrace ) -{ - using namespace bsoncxx::types; - using bsoncxx::builder::basic::kvp; - - if( executed && atrace.receiver == chain::config::system_account_name ) { - update_account( atrace.act ); - } - - bool added = false; - const bool in_filter = (store_action_traces || store_transaction_traces) && start_block_reached && - filter_include( atrace.receiver, atrace.act.name, atrace.act.authorization ); - write_ttrace |= in_filter; - if( start_block_reached && store_action_traces && in_filter ) { - auto action_traces_doc = bsoncxx::builder::basic::document{}; - // improve data distributivity when using mongodb sharding - action_traces_doc.append( kvp( "_id", make_custom_oid() ) ); - - auto v = to_variant_with_abi( atrace ); - try { - action_traces_doc.append( bsoncxx::builder::concatenate_doc{to_bson( v )} ); - } catch( bsoncxx::exception& e ) { - elog( "Unable to convert action trace to BSON: ${e}", ("e", e.what()) ); - try { - elog( " JSON: ${j}", ("j", fc::json::to_string( v, fc::time_point::now() + fc::exception::format_time_limit )) ); - } catch(...) {} - } - if( t->receipt.valid() ) { - action_traces_doc.append( kvp( "trx_status", std::string( t->receipt->status ) ) ); - } - action_traces_doc.append( kvp( "createdAt", b_date{now} ) ); - - mongocxx::model::insert_one insert_op{action_traces_doc.view()}; - bulk_action_traces.append( insert_op ); - added = true; - } - - return added; -} - - -void mongo_db_plugin_impl::_process_applied_transaction( const chain::transaction_trace_ptr& t ) { - using namespace bsoncxx::types; - using bsoncxx::builder::basic::kvp; - - auto trans_traces_doc = bsoncxx::builder::basic::document{}; - - auto now = std::chrono::duration_cast( - std::chrono::microseconds{fc::time_point::now().time_since_epoch().count()}); - - mongocxx::options::bulk_write bulk_opts; - bulk_opts.ordered(false); - mongocxx::bulk_write bulk_action_traces = _action_traces.create_bulk_write(bulk_opts); - bool write_atraces = false; - bool write_ttrace = false; // filters apply to transaction_traces as well - bool executed = t->receipt.valid() && t->receipt->status == chain::transaction_receipt_header::executed; - - for( const auto& atrace : t->action_traces ) { - try { - write_atraces |= add_action_trace( bulk_action_traces, atrace, t, executed, now, write_ttrace ); - } catch(...) { - handle_mongo_exception("add action traces", __LINE__); - } - } - - if( !start_block_reached ) return; //< add_action_trace calls update_account which must be called always - - // transaction trace insert - - if( store_transaction_traces && write_ttrace ) { - try { - auto v = to_variant_with_abi( *t ); - try { - trans_traces_doc.append( bsoncxx::builder::concatenate_doc{to_bson( v )} ); - } catch( bsoncxx::exception& e ) { - elog( "Unable to convert transaction to BSON: ${e}", ("e", e.what()) ); - try { - elog( " JSON: ${j}", ("j", fc::json::to_string( v, fc::time_point::now() + fc::exception::format_time_limit )) ); - } catch(...) {} - } - trans_traces_doc.append( kvp( "createdAt", b_date{now} ) ); - - try { - if( !_trans_traces.insert_one( trans_traces_doc.view() ) ) { - EOS_ASSERT( false, chain::mongo_db_insert_fail, "Failed to insert trans ${id}", ("id", t->id) ); - } - } catch( ... ) { - handle_mongo_exception( "trans_traces insert: " + t->id.str(), __LINE__ ); - } - } catch( ... ) { - handle_mongo_exception( "trans_traces serialization: " + t->id.str(), __LINE__ ); - } - } - - // insert action_traces - if( write_atraces ) { - try { - if( !bulk_action_traces.execute() ) { - EOS_ASSERT( false, chain::mongo_db_insert_fail, - "Bulk action traces insert failed for transaction trace: ${id}", ("id", t->id) ); - } - } catch( ... ) { - handle_mongo_exception( "action traces insert", __LINE__ ); - } - } - -} - -void mongo_db_plugin_impl::_process_accepted_block( const chain::block_state_ptr& bs ) { - using namespace bsoncxx::types; - using namespace bsoncxx::builder; - using bsoncxx::builder::basic::kvp; - using bsoncxx::builder::basic::make_document; - - mongocxx::options::update update_opts{}; - update_opts.upsert( true ); - - auto block_num = bs->block_num; - if( block_num % 1000 == 0 ) - ilog( "block_num: ${b}", ("b", block_num) ); - const auto& block_id = bs->id; - const auto block_id_str = block_id.str(); - - auto now = std::chrono::duration_cast( - std::chrono::microseconds{fc::time_point::now().time_since_epoch().count()}); - - if( store_block_states ) { - auto block_state_doc = bsoncxx::builder::basic::document{}; - block_state_doc.append( kvp( "block_num", b_int32{static_cast(block_num)} ), - kvp( "block_id", block_id_str ), - kvp( "validated", b_bool{true} ) ); - - const chain::block_header_state& bhs = *bs; - - try { - block_state_doc.append( kvp( "block_header_state", to_bson( fc::variant(bhs) ) ) ); - } catch( bsoncxx::exception& e ) { - elog( "Unable to convert block_header_state to BSON: ${e}", ("e", e.what()) ); - try { - elog( " JSON: ${j}", ("j", fc::json::to_string( bhs, fc::time_point::now() + fc::exception::format_time_limit )) ); - } catch(...) {} - } - block_state_doc.append( kvp( "createdAt", b_date{now} ) ); - - try { - if( update_blocks_via_block_num ) { - if( !_block_states.update_one( make_document( kvp( "block_num", b_int32{static_cast(block_num)} ) ), - make_document( kvp( "$set", block_state_doc.view() ) ), update_opts ) ) { - EOS_ASSERT( false, chain::mongo_db_insert_fail, "Failed to insert block_state ${num}", ("num", block_num) ); - } - } else { - if( !_block_states.update_one( make_document( kvp( "block_id", block_id_str ) ), - make_document( kvp( "$set", block_state_doc.view() ) ), update_opts ) ) { - EOS_ASSERT( false, chain::mongo_db_insert_fail, "Failed to insert block_state ${bid}", ("bid", block_id) ); - } - } - } catch( ... ) { - handle_mongo_exception( "block_states insert: " + block_id_str, __LINE__ ); - } - } - - if( store_blocks ) { - auto block_doc = bsoncxx::builder::basic::document{}; - block_doc.append( kvp( "block_num", b_int32{static_cast(block_num)} ), - kvp( "block_id", block_id_str ) ); - - auto v = to_variant_with_abi( *bs->block ); - try { - block_doc.append( kvp( "block", to_bson( v ) ) ); - } catch( bsoncxx::exception& e ) { - elog( "Unable to convert block to BSON: ${e}", ("e", e.what()) ); - try { - elog( " JSON: ${j}", ("j", fc::json::to_string( v, fc::time_point::now() + fc::exception::format_time_limit )) ); - } catch(...) {} - } - block_doc.append( kvp( "createdAt", b_date{now} ) ); - - try { - if( update_blocks_via_block_num ) { - if( !_blocks.update_one( make_document( kvp( "block_num", b_int32{static_cast(block_num)} ) ), - make_document( kvp( "$set", block_doc.view() ) ), update_opts ) ) { - EOS_ASSERT( false, chain::mongo_db_insert_fail, "Failed to insert block ${num}", ("num", block_num) ); - } - } else { - if( !_blocks.update_one( make_document( kvp( "block_id", block_id_str ) ), - make_document( kvp( "$set", block_doc.view() ) ), update_opts ) ) { - EOS_ASSERT( false, chain::mongo_db_insert_fail, "Failed to insert block ${bid}", ("bid", block_id) ); - } - } - } catch( ... ) { - handle_mongo_exception( "blocks insert: " + block_id_str, __LINE__ ); - } - } -} - -void mongo_db_plugin_impl::_process_irreversible_block(const chain::block_state_ptr& bs) -{ - using namespace bsoncxx::types; - using namespace bsoncxx::builder; - using bsoncxx::builder::basic::make_document; - using bsoncxx::builder::basic::kvp; - - - const auto block_id = bs->block->id(); - const auto block_id_str = block_id.str(); - - auto now = std::chrono::duration_cast( - std::chrono::microseconds{fc::time_point::now().time_since_epoch().count()}); - - if( store_blocks ) { - auto ir_block = find_block( _blocks, block_id_str ); - if( !ir_block ) { - _process_accepted_block( bs ); - ir_block = find_block( _blocks, block_id_str ); - if( !ir_block ) return; // should never happen - } - - auto update_doc = make_document( kvp( "$set", make_document( kvp( "irreversible", b_bool{true} ), - kvp( "updatedAt", b_date{now} ) ) ) ); - - _blocks.update_one( make_document( kvp( "_id", ir_block->view()["_id"].get_oid() ) ), update_doc.view() ); - } - - if( store_block_states ) { - auto ir_block = find_block( _block_states, block_id_str ); - if( !ir_block ) { - _process_accepted_block( bs ); - ir_block = find_block( _block_states, block_id_str ); - if( !ir_block ) return; // should never happen - } - - auto update_doc = make_document( kvp( "$set", make_document( kvp( "irreversible", b_bool{true} ), - kvp( "updatedAt", b_date{now} ) ) ) ); - - _block_states.update_one( make_document( kvp( "_id", ir_block->view()["_id"].get_oid() ) ), update_doc.view() ); - } - - if( store_transactions ) { - const auto block_num = bs->block->block_num(); - bool transactions_in_block = false; - mongocxx::options::bulk_write bulk_opts; - bulk_opts.ordered( false ); - auto bulk = _trans.create_bulk_write( bulk_opts ); - - for( const auto& receipt : bs->block->transactions ) { - string trx_id_str; - if( receipt.trx.contains() ) { - const auto& pt = receipt.trx.get(); - if( !filter_include( pt.get_signed_transaction() ) ) continue; - const auto& id = pt.id(); - trx_id_str = id.str(); - } else { - const auto& id = receipt.trx.get(); - trx_id_str = id.str(); - } - - auto update_doc = make_document( kvp( "$set", make_document( kvp( "irreversible", b_bool{true} ), - kvp( "block_id", block_id_str ), - kvp( "block_num", b_int32{static_cast(block_num)} ), - kvp( "updatedAt", b_date{now} ) ) ) ); - - mongocxx::model::update_one update_op{make_document( kvp( "trx_id", trx_id_str ) ), update_doc.view()}; - update_op.upsert( false ); - bulk.append( update_op ); - transactions_in_block = true; - } - - if( transactions_in_block ) { - try { - if( !bulk.execute() ) { - EOS_ASSERT( false, chain::mongo_db_insert_fail, "Bulk transaction insert failed for block: ${bid}", ("bid", block_id) ); - } - } catch( ... ) { - handle_mongo_exception( "bulk transaction insert", __LINE__ ); - } - } - } -} - -void mongo_db_plugin_impl::add_pub_keys( const vector& keys, const account_name& name, - const permission_name& permission, const std::chrono::milliseconds& now ) -{ - using bsoncxx::builder::basic::kvp; - using bsoncxx::builder::basic::make_document; - using namespace bsoncxx::types; - - if( keys.empty()) return; - - mongocxx::bulk_write bulk = _pub_keys.create_bulk_write(); - - for( const auto& pub_key_weight : keys ) { - auto find_doc = bsoncxx::builder::basic::document(); - - find_doc.append( kvp( "account", name.to_string()), - kvp( "public_key", pub_key_weight.key.to_string()), - kvp( "permission", permission.to_string()) ); - - auto update_doc = make_document( kvp( "$set", make_document( bsoncxx::builder::concatenate_doc{find_doc.view()}, - kvp( "createdAt", b_date{now} )))); - - mongocxx::model::update_one insert_op{find_doc.view(), update_doc.view()}; - insert_op.upsert(true); - bulk.append( insert_op ); - } - - try { - if( !bulk.execute()) { - EOS_ASSERT( false, chain::mongo_db_insert_fail, - "Bulk pub_keys insert failed for account: ${a}, permission: ${p}", - ("a", name)( "p", permission )); - } - } catch (...) { - handle_mongo_exception( "pub_keys insert", __LINE__ ); - } -} - -void mongo_db_plugin_impl::remove_pub_keys( const account_name& name, const permission_name& permission ) -{ - using bsoncxx::builder::basic::kvp; - using bsoncxx::builder::basic::make_document; - - try { - auto result = _pub_keys.delete_many( make_document( kvp( "account", name.to_string()), - kvp( "permission", permission.to_string()))); - if( !result ) { - EOS_ASSERT( false, chain::mongo_db_update_fail, - "pub_keys delete failed for account: ${a}, permission: ${p}", - ("a", name)( "p", permission )); - } - } catch (...) { - handle_mongo_exception( "pub_keys delete", __LINE__ ); - } -} - -void mongo_db_plugin_impl::add_account_control( const vector& controlling_accounts, - const account_name& name, const permission_name& permission, - const std::chrono::milliseconds& now ) -{ - using bsoncxx::builder::basic::kvp; - using bsoncxx::builder::basic::make_document; - using namespace bsoncxx::types; - - if( controlling_accounts.empty()) return; - - mongocxx::bulk_write bulk = _account_controls.create_bulk_write(); - - for( const auto& controlling_account : controlling_accounts ) { - auto find_doc = bsoncxx::builder::basic::document(); - - find_doc.append( kvp( "controlled_account", name.to_string()), - kvp( "controlled_permission", permission.to_string()), - kvp( "controlling_account", controlling_account.permission.actor.to_string()) ); - - auto update_doc = make_document( kvp( "$set", make_document( bsoncxx::builder::concatenate_doc{find_doc.view()}, - kvp( "createdAt", b_date{now} )))); - - - mongocxx::model::update_one insert_op{find_doc.view(), update_doc.view()}; - insert_op.upsert(true); - bulk.append( insert_op ); - } - - try { - if( !bulk.execute()) { - EOS_ASSERT( false, chain::mongo_db_insert_fail, - "Bulk account_controls insert failed for account: ${a}, permission: ${p}", - ("a", name)( "p", permission )); - } - } catch (...) { - handle_mongo_exception( "account_controls insert", __LINE__ ); - } -} - -void mongo_db_plugin_impl::remove_account_control( const account_name& name, const permission_name& permission ) -{ - using bsoncxx::builder::basic::kvp; - using bsoncxx::builder::basic::make_document; - - try { - auto result = _account_controls.delete_many( make_document( kvp( "controlled_account", name.to_string()), - kvp( "controlled_permission", permission.to_string()))); - if( !result ) { - EOS_ASSERT( false, chain::mongo_db_update_fail, - "account_controls delete failed for account: ${a}, permission: ${p}", - ("a", name)( "p", permission )); - } - } catch (...) { - handle_mongo_exception( "account_controls delete", __LINE__ ); - } -} - -namespace { - -void create_account( mongocxx::collection& accounts, const name& name, std::chrono::milliseconds& now ) { - using namespace bsoncxx::types; - using bsoncxx::builder::basic::kvp; - using bsoncxx::builder::basic::make_document; - - mongocxx::options::update update_opts{}; - update_opts.upsert( true ); - - const string name_str = name.to_string(); - auto update = make_document( - kvp( "$set", make_document( kvp( "name", name_str), - kvp( "createdAt", b_date{now} )))); - try { - if( !accounts.update_one( make_document( kvp( "name", name_str )), update.view(), update_opts )) { - EOS_ASSERT( false, chain::mongo_db_update_fail, "Failed to insert account ${n}", ("n", name)); - } - } catch (...) { - handle_mongo_exception( "create_account", __LINE__ ); - } -} - -} - -void mongo_db_plugin_impl::update_account(const chain::action& act) -{ - using bsoncxx::builder::basic::kvp; - using bsoncxx::builder::basic::make_document; - using namespace bsoncxx::types; - - if (act.account != chain::config::system_account_name) - return; - - try { - if( act.name == newaccount ) { - std::chrono::milliseconds now = std::chrono::duration_cast( - std::chrono::microseconds{fc::time_point::now().time_since_epoch().count()} ); - auto newacc = act.data_as(); - - create_account( _accounts, newacc.name, now ); - - add_pub_keys( newacc.owner.keys, newacc.name, owner, now ); - add_account_control( newacc.owner.accounts, newacc.name, owner, now ); - add_pub_keys( newacc.active.keys, newacc.name, active, now ); - add_account_control( newacc.active.accounts, newacc.name, active, now ); - - } else if( act.name == updateauth ) { - auto now = std::chrono::duration_cast( - std::chrono::microseconds{fc::time_point::now().time_since_epoch().count()} ); - const auto update = act.data_as(); - remove_pub_keys(update.account, update.permission); - remove_account_control(update.account, update.permission); - add_pub_keys(update.auth.keys, update.account, update.permission, now); - add_account_control(update.auth.accounts, update.account, update.permission, now); - - } else if( act.name == deleteauth ) { - const auto del = act.data_as(); - remove_pub_keys( del.account, del.permission ); - remove_account_control(del.account, del.permission); - - } else if( act.name == setabi ) { - auto now = std::chrono::duration_cast( - std::chrono::microseconds{fc::time_point::now().time_since_epoch().count()} ); - auto setabi = act.data_as(); - - abi_cache_index.erase( setabi.account ); - - auto account = find_account( _accounts, setabi.account ); - if( !account ) { - create_account( _accounts, setabi.account, now ); - account = find_account( _accounts, setabi.account ); - } - if( account ) { - abi_def abi_def = fc::raw::unpack( setabi.abi ); - auto v = fc::variant( abi_def ); - - try { - auto update_from = make_document( - kvp( "$set", make_document( kvp( "abi", to_bson( v )), - kvp( "updatedAt", b_date{now} )))); - - try { - if( !_accounts.update_one( make_document( kvp( "_id", account->view()["_id"].get_oid())), - update_from.view())) { - EOS_ASSERT( false, chain::mongo_db_update_fail, "Failed to udpdate account ${n}", ("n", setabi.account)); - } - } catch( ... ) { - handle_mongo_exception( "account update", __LINE__ ); - } - } catch( bsoncxx::exception& e ) { - elog( "Unable to convert abi JSON to BSON: ${e}", ("e", e.what())); - try { - elog( " JSON: ${j}", ("j", fc::json::to_string( v, fc::time_point::now() + fc::exception::format_time_limit )) ); - } catch(...) {} - } - } - } - } catch( fc::exception& e ) { - // if unable to unpack native type, skip account creation - } -} - -mongo_db_plugin_impl::mongo_db_plugin_impl() -{ -} - -mongo_db_plugin_impl::~mongo_db_plugin_impl() { - if (!startup) { - try { - ilog( "mongo_db_plugin shutdown in process please be patient this can take a few minutes" ); - done = true; - condition.notify_one(); - - consume_thread.join(); - - mongo_pool.reset(); - } catch( std::exception& e ) { - elog( "Exception on mongo_db_plugin shutdown of consume thread: ${e}", ("e", e.what())); - } - } -} - -void mongo_db_plugin_impl::wipe_database() { - ilog("mongo db wipe_database"); - - auto client = mongo_pool->acquire(); - auto& mongo_conn = *client; - - auto block_states = mongo_conn[db_name][block_states_col]; - auto blocks = mongo_conn[db_name][blocks_col]; - auto trans = mongo_conn[db_name][trans_col]; - auto trans_traces = mongo_conn[db_name][trans_traces_col]; - auto action_traces = mongo_conn[db_name][action_traces_col]; - auto accounts = mongo_conn[db_name][accounts_col]; - auto pub_keys = mongo_conn[db_name][pub_keys_col]; - auto account_controls = mongo_conn[db_name][account_controls_col]; - - block_states.drop(); - blocks.drop(); - trans.drop(); - trans_traces.drop(); - action_traces.drop(); - accounts.drop(); - pub_keys.drop(); - account_controls.drop(); - ilog("done wipe_database"); -} - -void mongo_db_plugin_impl::create_expiration_index(mongocxx::collection& collection, uint32_t expire_after_seconds) { - using bsoncxx::builder::basic::make_document; - using bsoncxx::builder::basic::kvp; - - auto indexes = collection.indexes(); - for( auto& index : indexes.list()) { - auto key = index["key"]; - if( !key ) { - continue; - } - auto field = key["createdAt"]; - if( !field ) { - continue; - } - - auto ttl = index["expireAfterSeconds"]; - if( ttl && ttl.get_int32() == expire_after_seconds ) { - return; - } else { - auto name = index["name"].get_utf8(); - ilog( "mongo db drop ttl index for collection ${collection}", ( "collection", collection.name().to_string())); - indexes.drop_one( name.value ); - break; - } - } - - mongocxx::options::index index_options{}; - index_options.expire_after( std::chrono::seconds( expire_after_seconds )); - index_options.background( true ); - ilog( "mongo db create ttl index for collection ${collection}", ( "collection", collection.name().to_string())); - collection.create_index( make_document( kvp( "createdAt", 1 )), index_options ); -} - -void mongo_db_plugin_impl::init() { - using namespace bsoncxx::types; - using bsoncxx::builder::basic::make_document; - using bsoncxx::builder::basic::kvp; - // Create the native contract accounts manually; sadly, we can't run their contracts to make them create themselves - // See native_contract_chain_initializer::prepare_database() - - ilog("init mongo"); - try { - auto client = mongo_pool->acquire(); - auto& mongo_conn = *client; - - auto accounts = mongo_conn[db_name][accounts_col]; - if( accounts.estimated_document_count() == 0 ) { - auto now = std::chrono::duration_cast( - std::chrono::microseconds{fc::time_point::now().time_since_epoch().count()} ); - - auto doc = make_document( kvp( "name", name( chain::config::system_account_name ).to_string()), - kvp( "createdAt", b_date{now} )); - - try { - if( !accounts.insert_one( doc.view())) { - EOS_ASSERT( false, chain::mongo_db_insert_fail, "Failed to insert account ${n}", - ("n", name( chain::config::system_account_name ).to_string())); - } - } catch (...) { - handle_mongo_exception( "account insert", __LINE__ ); - } - - try { - // MongoDB administrators (to enable sharding) : - // 1. enableSharding database (default to EOS) - // 2. shardCollection: blocks, action_traces, transaction_traces, especially action_traces - // 3. Compound index with shard key (default to _id below), to improve query performance. - - // blocks indexes - auto blocks = mongo_conn[db_name][blocks_col]; - blocks.create_index( bsoncxx::from_json( R"xxx({ "block_num" : 1, "_id" : 1 })xxx" )); - blocks.create_index( bsoncxx::from_json( R"xxx({ "block_id" : 1, "_id" : 1 })xxx" )); - - auto block_states = mongo_conn[db_name][block_states_col]; - block_states.create_index( bsoncxx::from_json( R"xxx({ "block_num" : 1, "_id" : 1 })xxx" )); - block_states.create_index( bsoncxx::from_json( R"xxx({ "block_id" : 1, "_id" : 1 })xxx" )); - - // accounts indexes - accounts.create_index( bsoncxx::from_json( R"xxx({ "name" : 1, "_id" : 1 })xxx" )); - - // transactions indexes - auto trans = mongo_conn[db_name][trans_col]; - trans.create_index( bsoncxx::from_json( R"xxx({ "trx_id" : 1, "_id" : 1 })xxx" )); - - auto trans_trace = mongo_conn[db_name][trans_traces_col]; - trans_trace.create_index( bsoncxx::from_json( R"xxx({ "id" : 1, "_id" : 1 })xxx" )); - - // action traces indexes - auto action_traces = mongo_conn[db_name][action_traces_col]; - action_traces.create_index( bsoncxx::from_json( R"xxx({ "block_num" : 1, "_id" : 1 })xxx" )); - - // pub_keys indexes - auto pub_keys = mongo_conn[db_name][pub_keys_col]; - pub_keys.create_index( bsoncxx::from_json( R"xxx({ "account" : 1, "permission" : 1, "_id" : 1 })xxx" )); - pub_keys.create_index( bsoncxx::from_json( R"xxx({ "public_key" : 1, "_id" : 1 })xxx" )); - - // account_controls indexes - auto account_controls = mongo_conn[db_name][account_controls_col]; - account_controls.create_index( - bsoncxx::from_json( R"xxx({ "controlled_account" : 1, "controlled_permission" : 1, "_id" : 1 })xxx" )); - account_controls.create_index( bsoncxx::from_json( R"xxx({ "controlling_account" : 1, "_id" : 1 })xxx" )); - - } catch (...) { - handle_mongo_exception( "create indexes", __LINE__ ); - } - } - - if( expire_after_seconds > 0 ) { - try { - mongocxx::collection block_states = mongo_conn[db_name][block_states_col]; - create_expiration_index( block_states, expire_after_seconds ); - mongocxx::collection blocks = mongo_conn[db_name][blocks_col]; - create_expiration_index( blocks, expire_after_seconds ); - mongocxx::collection trans = mongo_conn[db_name][trans_col]; - create_expiration_index( trans, expire_after_seconds ); - mongocxx::collection trans_traces = mongo_conn[db_name][trans_traces_col]; - create_expiration_index( trans_traces, expire_after_seconds ); - mongocxx::collection action_traces = mongo_conn[db_name][action_traces_col]; - create_expiration_index( action_traces, expire_after_seconds ); - } catch(...) { - handle_mongo_exception( "create expiration indexes", __LINE__ ); - } - } - } catch (...) { - handle_mongo_exception( "mongo init", __LINE__ ); - } - - ilog("starting db plugin thread"); - - consume_thread = std::thread( [this] { - fc::set_os_thread_name( "mongodb" ); - consume_blocks(); - } ); - - startup = false; -} - -//////////// -// mongo_db_plugin -//////////// - -mongo_db_plugin::mongo_db_plugin() -:my(new mongo_db_plugin_impl) -{ -} - -mongo_db_plugin::~mongo_db_plugin() -{ -} - -void mongo_db_plugin::set_program_options(options_description& cli, options_description& cfg) -{ - cfg.add_options() - ("mongodb-queue-size,q", bpo::value()->default_value(1024), - "The target queue size between nodeos and MongoDB plugin thread.") - ("mongodb-abi-cache-size", bpo::value()->default_value(2048), - "The maximum size of the abi cache for serializing data.") - ("mongodb-wipe", bpo::bool_switch()->default_value(false), - "Required with --replay-blockchain, --hard-replay-blockchain, or --delete-all-blocks to wipe mongo db." - "This option required to prevent accidental wipe of mongo db.") - ("mongodb-block-start", bpo::value()->default_value(0), - "If specified then only abi data pushed to mongodb until specified block is reached.") - ("mongodb-uri,m", bpo::value(), - "MongoDB URI connection string, see: https://docs.mongodb.com/master/reference/connection-string/." - " If not specified then plugin is disabled. Default database 'EOS' is used if not specified in URI." - " Example: mongodb://127.0.0.1:27017/EOS") - ("mongodb-update-via-block-num", bpo::value()->default_value(false), - "Update blocks/block_state with latest via block number so that duplicates are overwritten.") - ("mongodb-store-blocks", bpo::value()->default_value(true), - "Enables storing blocks in mongodb.") - ("mongodb-store-block-states", bpo::value()->default_value(true), - "Enables storing block state in mongodb.") - ("mongodb-store-transactions", bpo::value()->default_value(true), - "Enables storing transactions in mongodb.") - ("mongodb-store-transaction-traces", bpo::value()->default_value(true), - "Enables storing transaction traces in mongodb.") - ("mongodb-store-action-traces", bpo::value()->default_value(true), - "Enables storing action traces in mongodb.") - ("mongodb-expire-after-seconds", bpo::value()->default_value(0), - "Enables expiring data in mongodb after a specified number of seconds.") - ("mongodb-filter-on", bpo::value>()->composing(), - "Track actions which match receiver:action:actor. Receiver, Action, & Actor may be blank to include all. i.e. eosio:: or :transfer: Use * or leave unspecified to include all.") - ("mongodb-filter-out", bpo::value>()->composing(), - "Do not track actions which match receiver:action:actor. Receiver, Action, & Actor may be blank to exclude all.") - ; -} - -void mongo_db_plugin::plugin_initialize(const variables_map& options) -{ - try { - if( options.count( "mongodb-uri" )) { - ilog( "initializing mongo_db_plugin" ); - my->configured = true; - - if( options.at( "replay-blockchain" ).as() || options.at( "hard-replay-blockchain" ).as() || options.at( "delete-all-blocks" ).as() ) { - if( options.at( "mongodb-wipe" ).as()) { - ilog( "Wiping mongo database on startup" ); - my->wipe_database_on_startup = true; - } else if( options.count( "mongodb-block-start" ) == 0 ) { - EOS_ASSERT( false, chain::plugin_config_exception, "--mongodb-wipe required with --replay-blockchain, --hard-replay-blockchain, or --delete-all-blocks" - " --mongodb-wipe will remove all EOS collections from mongodb." ); - } - } - - if( options.count( "abi-serializer-max-time-ms") == 0 ) { - EOS_ASSERT(false, chain::plugin_config_exception, "--abi-serializer-max-time-ms required as default value not appropriate for parsing full blocks"); - } - my->abi_serializer_max_time = app().get_plugin().get_abi_serializer_max_time(); - - if( options.count( "mongodb-queue-size" )) { - my->max_queue_size = options.at( "mongodb-queue-size" ).as(); - } - if( options.count( "mongodb-abi-cache-size" )) { - my->abi_cache_size = options.at( "mongodb-abi-cache-size" ).as(); - EOS_ASSERT( my->abi_cache_size > 0, chain::plugin_config_exception, "mongodb-abi-cache-size > 0 required" ); - } - if( options.count( "mongodb-block-start" )) { - my->start_block_num = options.at( "mongodb-block-start" ).as(); - } - if( options.count( "mongodb-update-via-block-num" )) { - my->update_blocks_via_block_num = options.at( "mongodb-update-via-block-num" ).as(); - } - if( options.count( "mongodb-store-blocks" )) { - my->store_blocks = options.at( "mongodb-store-blocks" ).as(); - } - if( options.count( "mongodb-store-block-states" )) { - my->store_block_states = options.at( "mongodb-store-block-states" ).as(); - } - if( options.count( "mongodb-store-transactions" )) { - my->store_transactions = options.at( "mongodb-store-transactions" ).as(); - } - if( options.count( "mongodb-store-transaction-traces" )) { - my->store_transaction_traces = options.at( "mongodb-store-transaction-traces" ).as(); - } - if( options.count( "mongodb-store-action-traces" )) { - my->store_action_traces = options.at( "mongodb-store-action-traces" ).as(); - } - if( options.count( "mongodb-expire-after-seconds" )) { - my->expire_after_seconds = options.at( "mongodb-expire-after-seconds" ).as(); - } - if( options.count( "mongodb-filter-on" )) { - auto fo = options.at( "mongodb-filter-on" ).as>(); - my->filter_on_star = false; - for( auto& s : fo ) { - if( s == "*" ) { - my->filter_on_star = true; - break; - } - std::vector v; - boost::split( v, s, boost::is_any_of( ":" )); - EOS_ASSERT( v.size() == 3, fc::invalid_arg_exception, "Invalid value ${s} for --mongodb-filter-on", ("s", s)); - filter_entry fe{eosio::chain::name(v[0]), eosio::chain::name(v[1]), eosio::chain::name(v[2])}; - my->filter_on.insert( fe ); - } - } else { - my->filter_on_star = true; - } - if( options.count( "mongodb-filter-out" )) { - auto fo = options.at( "mongodb-filter-out" ).as>(); - for( auto& s : fo ) { - std::vector v; - boost::split( v, s, boost::is_any_of( ":" )); - EOS_ASSERT( v.size() == 3, fc::invalid_arg_exception, "Invalid value ${s} for --mongodb-filter-out", ("s", s)); - filter_entry fe{eosio::chain::name(v[0]), eosio::chain::name(v[1]), eosio::chain::name(v[2])}; - my->filter_out.insert( fe ); - } - } - if( options.count( "producer-name") ) { - wlog( "mongodb plugin not recommended on producer node" ); - my->is_producer = true; - } - - if( my->start_block_num == 0 ) { - my->start_block_reached = true; - } - - std::string uri_str = options.at( "mongodb-uri" ).as(); - ilog( "connecting to ${u}", ("u", uri_str)); - mongocxx::uri uri = mongocxx::uri{uri_str}; - my->db_name = uri.database(); - if( my->db_name.empty()) - my->db_name = "EOS"; - my->mongo_pool.emplace(uri); - - // hook up to signals on controller - chain_plugin* chain_plug = app().find_plugin(); - EOS_ASSERT( chain_plug, chain::missing_chain_plugin_exception, "" ); - auto& chain = chain_plug->chain(); - my->chain_id.emplace( chain.get_chain_id()); - - my->accepted_block_connection.emplace( chain.accepted_block.connect( [&]( const chain::block_state_ptr& bs ) { - my->accepted_block( bs ); - } )); - my->irreversible_block_connection.emplace( - chain.irreversible_block.connect( [&]( const chain::block_state_ptr& bs ) { - my->applied_irreversible_block( bs ); - } )); - my->accepted_transaction_connection.emplace( - chain.accepted_transaction.connect( [&]( const chain::transaction_metadata_ptr& t ) { - my->accepted_transaction( t ); - } )); - my->applied_transaction_connection.emplace( - chain.applied_transaction.connect( [&]( std::tuple t ) { - my->applied_transaction( std::get<0>(t) ); - } )); - - if( my->wipe_database_on_startup ) { - my->wipe_database(); - } - my->init(); - } else { - wlog( "eosio::mongo_db_plugin configured, but no --mongodb-uri specified." ); - wlog( "mongo_db_plugin disabled." ); - } - } FC_LOG_AND_RETHROW() -} - -void mongo_db_plugin::plugin_startup() -{ -} - -void mongo_db_plugin::plugin_shutdown() -{ - my->accepted_block_connection.reset(); - my->irreversible_block_connection.reset(); - my->accepted_transaction_connection.reset(); - my->applied_transaction_connection.reset(); - - my.reset(); -} - -} // namespace eosio - diff --git a/plugins/net_api_plugin/net.swagger.yaml b/plugins/net_api_plugin/net.swagger.yaml index 4bba46ef39e..d89f007009d 100644 --- a/plugins/net_api_plugin/net.swagger.yaml +++ b/plugins/net_api_plugin/net.swagger.yaml @@ -61,17 +61,17 @@ paths: description: Incremental value above a computed base type: integer chain_id: - $ref: 'https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml' + $ref: 'https://eosio.github.io/schemata/v2.1/oas/Sha256.yaml' node_id: - $ref: 'https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml' + $ref: 'https://eosio.github.io/schemata/v2.1/oas/Sha256.yaml' key: - $ref: 'https://eosio.github.io/schemata/v2.0/oas/PublicKey.yaml' + $ref: 'https://eosio.github.io/schemata/v2.1/oas/PublicKey.yaml' time: - $ref: 'https://eosio.github.io/schemata/v2.0/oas/DateTimeSeconds.yaml' + $ref: 'https://eosio.github.io/schemata/v2.1/oas/DateTimeSeconds.yaml' token: - $ref: 'https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml' + $ref: 'https://eosio.github.io/schemata/v2.1/oas/Sha256.yaml' sig: - $ref: 'https://eosio.github.io/schemata/v2.0/oas/Signature.yaml' + $ref: 'https://eosio.github.io/schemata/v2.1/oas/Signature.yaml' p2p_address: description: IP address or URL of the peer type: string @@ -79,12 +79,12 @@ paths: description: Last irreversible block number type: integer last_irreversible_block_id: - $ref: 'https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml' + $ref: 'https://eosio.github.io/schemata/v2.1/oas/Sha256.yaml' head_num: description: Head number type: integer head_id: - $ref: 'https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml' + $ref: 'https://eosio.github.io/schemata/v2.1/oas/Sha256.yaml' os: description: Operating system name type: string @@ -190,17 +190,17 @@ paths: description: Incremental value above a computed base type: integer chain_id: - $ref: 'https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml' + $ref: 'https://eosio.github.io/schemata/v2.1/oas/Sha256.yaml' node_id: - $ref: 'https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml' + $ref: 'https://eosio.github.io/schemata/v2.1/oas/Sha256.yaml' key: - $ref: 'https://eosio.github.io/schemata/v2.0/oas/PublicKey.yaml' + $ref: 'https://eosio.github.io/schemata/v2.1/oas/PublicKey.yaml' time: - $ref: 'https://eosio.github.io/schemata/v2.0/oas/DateTimeSeconds.yaml' + $ref: 'https://eosio.github.io/schemata/v2.1/oas/DateTimeSeconds.yaml' token: - $ref: 'https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml' + $ref: 'https://eosio.github.io/schemata/v2.1/oas/Sha256.yaml' sig: - $ref: 'https://eosio.github.io/schemata/v2.0/oas/Signature.yaml' + $ref: 'https://eosio.github.io/schemata/v2.1/oas/Signature.yaml' p2p_address: description: IP address or URL of the peer type: string @@ -208,12 +208,12 @@ paths: description: Last irreversible block number type: integer last_irreversible_block_id: - $ref: 'https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml' + $ref: 'https://eosio.github.io/schemata/v2.1/oas/Sha256.yaml' head_num: description: Head number type: integer head_id: - $ref: 'https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml' + $ref: 'https://eosio.github.io/schemata/v2.1/oas/Sha256.yaml' os: description: Operating system name type: string diff --git a/plugins/net_api_plugin/net_api_plugin.cpp b/plugins/net_api_plugin/net_api_plugin.cpp index 0d3a7ab6f42..ba8cb9cef84 100644 --- a/plugins/net_api_plugin/net_api_plugin.cpp +++ b/plugins/net_api_plugin/net_api_plugin.cpp @@ -19,11 +19,10 @@ static appbase::abstract_plugin& _net_api_plugin = app().register_plugin()); - -#define INVOKE_R_R_R_R(api_handle, call_name, in_param0, in_param1, in_param2) \ - const auto& vs = fc::json::json::from_string(body).as(); \ - auto result = api_handle.call_name(vs.at(0).as(), vs.at(1).as(), vs.at(2).as()); + auto params = parse_params(body);\ + fc::variant result( api_handle.call_name( std::move(params) ) ); #define INVOKE_R_V(api_handle, call_name) \ + body = parse_params(body); \ auto result = api_handle.call_name(); #define INVOKE_V_R(api_handle, call_name, in_param) \ - api_handle.call_name(fc::json::from_string(body).as()); \ - eosio::detail::net_api_plugin_empty result; - -#define INVOKE_V_R_R(api_handle, call_name, in_param0, in_param1) \ - const auto& vs = fc::json::json::from_string(body).as(); \ - api_handle.call_name(vs.at(0).as(), vs.at(1).as()); \ + auto params = parse_params(body);\ + api_handle.call_name( std::move(params) ); \ eosio::detail::net_api_plugin_empty result; #define INVOKE_V_V(api_handle, call_name) \ + body = parse_params(body); \ api_handle.call_name(); \ eosio::detail::net_api_plugin_empty result; @@ -64,13 +58,13 @@ void net_api_plugin::plugin_startup() { // INVOKE_V_R(net_mgr, set_timeout, int64_t), 200), // CALL(net, net_mgr, sign_transaction, // INVOKE_R_R_R_R(net_mgr, sign_transaction, chain::signed_transaction, flat_set, chain::chain_id_type), 201), - CALL(net, net_mgr, connect, + CALL_WITH_400(net, net_mgr, connect, INVOKE_R_R(net_mgr, connect, std::string), 201), - CALL(net, net_mgr, disconnect, + CALL_WITH_400(net, net_mgr, disconnect, INVOKE_R_R(net_mgr, disconnect, std::string), 201), - CALL(net, net_mgr, status, + CALL_WITH_400(net, net_mgr, status, INVOKE_R_R(net_mgr, status, std::string), 201), - CALL(net, net_mgr, connections, + CALL_WITH_400(net, net_mgr, connections, INVOKE_R_V(net_mgr, connections), 201), // CALL(net, net_mgr, open, // INVOKE_V_R(net_mgr, open, std::string), 200), @@ -95,10 +89,8 @@ void net_api_plugin::plugin_initialize(const variables_map& options) { #undef INVOKE_R_R -#undef INVOKE_R_R_R_R #undef INVOKE_R_V #undef INVOKE_V_R -#undef INVOKE_V_R_R #undef INVOKE_V_V #undef CALL diff --git a/plugins/net_plugin/include/eosio/net_plugin/net_plugin.hpp b/plugins/net_plugin/include/eosio/net_plugin/net_plugin.hpp index b7b3516e18f..750ac3a74a3 100644 --- a/plugins/net_plugin/include/eosio/net_plugin/net_plugin.hpp +++ b/plugins/net_plugin/include/eosio/net_plugin/net_plugin.hpp @@ -27,10 +27,10 @@ namespace eosio { void plugin_startup(); void plugin_shutdown(); - string connect( const string& endpoint ); - string disconnect( const string& endpoint ); - optional status( const string& endpoint )const; - vector connections()const; + string connect( const string& endpoint ); + string disconnect( const string& endpoint ); + std::optional status( const string& endpoint )const; + vector connections()const; private: std::shared_ptr my; diff --git a/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp b/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp index 56c33909851..f1800df75d1 100644 --- a/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp +++ b/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp @@ -135,15 +135,22 @@ namespace eosio { uint32_t end_block{0}; }; - using net_message = static_variant; // which = 8 + struct trx_message_v1 { + std::optional trx_id; // only provided for large trx as trade-off for small trxs not worth it + std::shared_ptr trx; + }; + + using net_message = std::variant; // which = 10 } // namespace eosio @@ -162,6 +169,8 @@ FC_REFLECT( eosio::time_message, (org)(rec)(xmt)(dst) ) FC_REFLECT( eosio::notice_message, (known_trx)(known_blocks) ) FC_REFLECT( eosio::request_message, (req_trx)(req_blocks) ) FC_REFLECT( eosio::sync_request_message, (start_block)(end_block) ) +FC_REFLECT( eosio::trx_message_v1, (trx_id)(trx) ) + /** * diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 17e9b19c224..c8fa261699c 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -97,7 +98,7 @@ namespace eosio { bool have_block = false; // true if we have received the block, false if only received id notification }; - struct by_block_id; + struct by_peer_block_id; typedef multi_index_container< eosio::peer_block_state, @@ -109,7 +110,7 @@ namespace eosio { >, composite_key_compare< std::less, sha256_less > >, - ordered_non_unique< tag, + ordered_non_unique< tag, composite_key< peer_block_state, member, member @@ -178,10 +179,9 @@ namespace eosio { explicit dispatch_manager(boost::asio::io_context& io_context) : strand( io_context ) {} - void bcast_transaction(const packed_transaction& trx); + void bcast_transaction(const packed_transaction_ptr& trx); void rejected_transaction(const packed_transaction_ptr& trx, uint32_t head_blk_num); void bcast_block( const signed_block_ptr& b, const block_id_type& id ); - void bcast_notice( const block_id_type& id ); void rejected_block(const block_id_type& id); void recv_block(const connection_ptr& conn, const block_id_type& msg, uint32_t bnum); @@ -195,6 +195,7 @@ namespace eosio { bool have_block(const block_id_type& blkid) const; bool add_peer_txn( const node_transaction_state& nts ); + bool add_peer_txn( const transaction_id_type& tid, uint32_t connection_id ); void update_txns_block_num( const signed_block_ptr& sb ); void update_txns_block_num( const transaction_id_type& id, uint32_t blk_num ); bool peer_has_txn( const transaction_id_type& tid, uint32_t connection_id ) const; @@ -232,12 +233,14 @@ namespace eosio { boost::asio::steady_timer::duration connector_period{0}; boost::asio::steady_timer::duration txn_exp_period{0}; boost::asio::steady_timer::duration resp_expected_period{0}; - boost::asio::steady_timer::duration keepalive_interval{std::chrono::seconds{32}}; + std::chrono::milliseconds keepalive_interval{std::chrono::milliseconds{32 * 1000}}; + std::chrono::milliseconds heartbeat_timeout{keepalive_interval * 2}; int max_cleanup_time_ms = 0; uint32_t max_client_count = 0; uint32_t max_nodes_per_host = 1; bool p2p_accept_transactions = true; + bool p2p_reject_incomplete_blocks = true; /// Peer clock may be no more than 1 second skewed from our clock, including network latency. const std::chrono::system_clock::duration peer_authentication_interval{std::chrono::seconds{1}}; @@ -268,8 +271,8 @@ namespace eosio { compat::channels::transaction_ack::channel_type::handle incoming_transaction_ack_subscription; - uint16_t thread_pool_size = 2; - optional thread_pool; + uint16_t thread_pool_size = 2; + std::optional thread_pool; private: mutable std::mutex chain_info_mtx; // protects chain_* @@ -330,7 +333,7 @@ namespace eosio { */ chain::signature_type sign_compact(const chain::public_key_type& signer, const fc::sha256& digest) const; - constexpr uint16_t to_protocol_version(uint16_t v); + constexpr static uint16_t to_protocol_version(uint16_t v); connection_ptr find_connection(const string& host)const; // must call with held mutex }; @@ -387,10 +390,13 @@ namespace eosio { constexpr auto def_txn_expire_wait = std::chrono::seconds(3); constexpr auto def_resp_expected_wait = std::chrono::seconds(5); constexpr auto def_sync_fetch_span = 100; + constexpr auto def_keepalive_interval = 32000; constexpr auto message_header_size = 4; - constexpr uint32_t signed_block_which = 7; // see protocol net_message - constexpr uint32_t packed_transaction_which = 8; // see protocol net_message + constexpr uint32_t signed_block_v0_which = fc::get_index(); // see protocol net_message + constexpr uint32_t packed_transaction_v0_which = fc::get_index(); // see protocol net_message + constexpr uint32_t signed_block_which = fc::get_index(); // see protocol net_message + constexpr uint32_t trx_message_v1_which = fc::get_index(); // see protocol net_message /** * For a while, network version was a 16 bit value equal to the second set of 16 bits @@ -411,10 +417,13 @@ namespace eosio { * the need for compatibility hooks */ constexpr uint16_t proto_base = 0; - constexpr uint16_t proto_explicit_sync = 1; - constexpr uint16_t block_id_notify = 2; // reserved. feature was removed. next net_version should be 3 + constexpr uint16_t proto_explicit_sync = 1; // version at time of eosio 1.0 + constexpr uint16_t proto_block_id_notify = 2; // reserved. feature was removed. next net_version should be 3 + constexpr uint16_t proto_pruned_types = 3; // supports new signed_block & packed_transaction types + constexpr uint16_t heartbeat_interval = 4; // supports configurable heartbeat interval + constexpr uint16_t dup_goaway_resolution = 5; // support peer address based duplicate connection resolution - constexpr uint16_t net_version = proto_explicit_sync; + constexpr uint16_t net_version = dup_goaway_resolution; /** * Index by start_block_num @@ -579,13 +588,17 @@ namespace eosio { void set_connection_type( const string& peer_addr ); bool is_transactions_only_connection()const { return connection_type == transactions_only; } bool is_blocks_only_connection()const { return connection_type == blocks_only; } + void set_heartbeat_timeout(std::chrono::milliseconds msec) { + std::chrono::system_clock::duration dur = msec; + hb_timeout = dur.count(); + } private: static const string unknown; void update_endpoints(); - optional peer_requested; // this peer is requesting info from us + std::optional peer_requested; // this peer is requesting info from us std::atomic socket_open{false}; @@ -612,7 +625,8 @@ namespace eosio { int16_t sent_handshake_count = 0; std::atomic connecting{true}; std::atomic syncing{false}; - uint16_t protocol_version = 0; + + std::atomic protocol_version = 0; uint16_t consecutive_rejected_blocks = 0; block_status_monitor block_status_monitor_; std::atomic consecutive_immediate_connection_close = 0; @@ -622,18 +636,18 @@ namespace eosio { std::atomic no_retry{no_reason}; - mutable std::mutex conn_mtx; //< mtx for last_req .. local_endpoint_port - optional last_req; - handshake_message last_handshake_recv; - handshake_message last_handshake_sent; - block_id_type fork_head; - uint32_t fork_head_num{0}; - fc::time_point last_close; - fc::sha256 conn_node_id; - string remote_endpoint_ip; - string remote_endpoint_port; - string local_endpoint_ip; - string local_endpoint_port; + mutable std::mutex conn_mtx; //< mtx for last_req .. local_endpoint_port + std::optional last_req; + handshake_message last_handshake_recv; + handshake_message last_handshake_sent; + block_id_type fork_head; + uint32_t fork_head_num{0}; + fc::time_point last_close; + fc::sha256 conn_node_id; + string remote_endpoint_ip; + string remote_endpoint_port; + string local_endpoint_ip; + string local_endpoint_port; connection_status get_status()const; @@ -647,6 +661,9 @@ namespace eosio { tstamp dst{0}; //!< destination timestamp tstamp xmt{0}; //!< transmit timestamp /** @} */ + // timestamp for the lastest message + tstamp latest_msg_time{0}; + tstamp hb_timeout; bool connected(); bool current(); @@ -656,6 +673,9 @@ namespace eosio { void close( bool reconnect = true, bool shutdown = false ); private: static void _close( connection* self, bool reconnect, bool shutdown ); // for easy capture + + bool process_next_block_message(uint32_t message_length); + bool process_next_trx_message(uint32_t message_length); public: bool populate_handshake( handshake_message& hello, bool force ); @@ -679,6 +699,9 @@ namespace eosio { /** \name Peer Timestamps * Time message handling */ + /** \brief Check heartbeat time and send Time_message + */ + void check_heartbeat( tstamp current_time ); /** \brief Populate and queue time_message */ void send_time(); @@ -967,7 +990,7 @@ namespace eosio { bool has_last_req = false; { std::lock_guard g_conn( self->conn_mtx ); - has_last_req = !!self->last_req; + has_last_req = self->last_req.has_value(); self->last_handshake_recv = handshake_message(); self->last_handshake_sent = handshake_message(); self->last_close = fc::time_point::now(); @@ -1112,6 +1135,28 @@ namespace eosio { }); } + // called from connection strand + void connection::check_heartbeat( tstamp current_time ) + { + if( protocol_version >= heartbeat_interval ) { + if( latest_msg_time > 0 && current_time > latest_msg_time + hb_timeout ) { + no_retry = benign_other; + if( !peer_address().empty() ) { + fc_wlog(logger, "heartbeat timed out for peer address ${adr}", ("adr", peer_address())); + close(true); // reconnect + } else { + { + std::lock_guard g_conn( conn_mtx ); + fc_wlog(logger, "heartbeat timed out from ${p} ${ag}", ("p", last_handshake_recv.p2p_address)("ag", last_handshake_recv.agent)); + } + close(false); // don't reconnect + } + return; + } + } + send_time(); + } + void connection::send_time() { time_message xpkt; xpkt.org = rec; @@ -1176,10 +1221,14 @@ namespace eosio { c->enqueue_sync_block(); c->do_queue_write(); - } catch( const std::exception& ex ) { - fc_elog( logger, "Exception in do_queue_write to ${p} ${s}", ("p", c->peer_name())( "s", ex.what() ) ); + } catch ( const std::bad_alloc& ) { + throw; + } catch ( const boost::interprocess::bad_alloc& ) { + throw; } catch( const fc::exception& ex ) { fc_elog( logger, "Exception in do_queue_write to ${p} ${s}", ("p", c->peer_name())( "s", ex.to_string() ) ); + } catch( const std::exception& ex ) { + fc_elog( logger, "Exception in do_queue_write to ${p} ${s}", ("p", c->peer_name())( "s", ex.what() ) ); } catch( ... ) { fc_elog( logger, "Exception in do_queue_write to ${p}", ("p", c->peer_name()) ); } @@ -1240,65 +1289,177 @@ namespace eosio { return true; } - void connection::enqueue( const net_message& m ) { - verify_strand_in_this_thread( strand, __func__, __LINE__ ); - go_away_reason close_after_send = no_reason; - if (m.contains()) { - close_after_send = m.get().reason; + //------------------------------------------------------------------------ + + using send_buffer_type = std::shared_ptr>; + + struct buffer_factory { + + /// caches result for subsequent calls, only provide same net_message instance for each invocation + const send_buffer_type& get_send_buffer( const net_message& m ) { + if( !send_buffer ) { + send_buffer = create_send_buffer( m ); + } + return send_buffer; } - const uint32_t payload_size = fc::raw::pack_size( m ); + protected: + send_buffer_type send_buffer; - const char* const header = reinterpret_cast(&payload_size); // avoid variable size encoding of uint32_t - constexpr size_t header_size = sizeof(payload_size); - static_assert( header_size == message_header_size, "invalid message_header_size" ); - const size_t buffer_size = header_size + payload_size; + protected: + static send_buffer_type create_send_buffer( const net_message& m ) { + const uint32_t payload_size = fc::raw::pack_size( m ); - auto send_buffer = std::make_shared>(buffer_size); - fc::datastream ds( send_buffer->data(), buffer_size); - ds.write( header, header_size ); - fc::raw::pack( ds, m ); + const char* const header = reinterpret_cast(&payload_size); // avoid variable size encoding of uint32_t + constexpr size_t header_size = sizeof(payload_size); + static_assert( header_size == message_header_size, "invalid message_header_size" ); + const size_t buffer_size = header_size + payload_size; - enqueue_buffer( send_buffer, close_after_send ); - } + auto send_buffer = std::make_shared>(buffer_size); + fc::datastream ds( send_buffer->data(), buffer_size); + ds.write( header, header_size ); + fc::raw::pack( ds, m ); - template< typename T> - static std::shared_ptr> create_send_buffer( uint32_t which, const T& v ) { - // match net_message static_variant pack - const uint32_t which_size = fc::raw::pack_size( unsigned_int( which ) ); - const uint32_t payload_size = which_size + fc::raw::pack_size( v ); + return send_buffer; + } - const char* const header = reinterpret_cast(&payload_size); // avoid variable size encoding of uint32_t - constexpr size_t header_size = sizeof( payload_size ); - static_assert( header_size == message_header_size, "invalid message_header_size" ); - const size_t buffer_size = header_size + payload_size; + template< typename T> + static send_buffer_type create_send_buffer( uint32_t which, const T& v ) { + // match net_message static_variant pack + const uint32_t which_size = fc::raw::pack_size( unsigned_int( which ) ); + const uint32_t payload_size = which_size + fc::raw::pack_size( v ); - auto send_buffer = std::make_shared>( buffer_size ); - fc::datastream ds( send_buffer->data(), buffer_size ); - ds.write( header, header_size ); - fc::raw::pack( ds, unsigned_int( which ) ); - fc::raw::pack( ds, v ); + const char* const header = reinterpret_cast(&payload_size); // avoid variable size encoding of uint32_t + constexpr size_t header_size = sizeof( payload_size ); + static_assert( header_size == message_header_size, "invalid message_header_size" ); + const size_t buffer_size = header_size + payload_size; - return send_buffer; - } + auto send_buffer = std::make_shared>( buffer_size ); + fc::datastream ds( send_buffer->data(), buffer_size ); + ds.write( header, header_size ); + fc::raw::pack( ds, unsigned_int( which ) ); + fc::raw::pack( ds, v ); - static std::shared_ptr> create_send_buffer( const signed_block_ptr& sb ) { - // this implementation is to avoid copy of signed_block to net_message - // matches which of net_message for signed_block - fc_dlog( logger, "sending block ${bn}", ("bn", sb->block_num()) ); - return create_send_buffer( signed_block_which, *sb ); - } + return send_buffer; + } + + }; + + struct block_buffer_factory : public buffer_factory { + + /// caches result for subsequent calls, only provide same signed_block_ptr instance for each invocation. + /// protocol_version can differ per invocation as buffer_factory potentially caches multiple send buffers. + const send_buffer_type& get_send_buffer( const signed_block_ptr& sb, uint16_t protocol_version ) { + if( protocol_version >= proto_pruned_types ) { + if( !send_buffer ) { + send_buffer = create_send_buffer( sb ); + } + return send_buffer; + } else { + if( !send_buffer_v0 ) { + const auto v0 = sb->to_signed_block_v0(); + if( !v0 ) return send_buffer_v0; + send_buffer_v0 = create_send_buffer( *v0 ); + } + return send_buffer_v0; + } + } - static std::shared_ptr> create_send_buffer( const packed_transaction& trx ) { - // this implementation is to avoid copy of packed_transaction to net_message - // matches which of net_message for packed_transaction - return create_send_buffer( packed_transaction_which, trx ); + private: + send_buffer_type send_buffer_v0; + + private: + + static std::shared_ptr> create_send_buffer( const signed_block_ptr& sb ) { + static_assert( signed_block_which == fc::get_index() ); + // this implementation is to avoid copy of signed_block to net_message + // matches which of net_message for signed_block + fc_dlog( logger, "sending block ${bn}", ("bn", sb->block_num()) ); + return buffer_factory::create_send_buffer( signed_block_which, *sb ); + } + + static std::shared_ptr> create_send_buffer( const signed_block_v0& sb_v0 ) { + static_assert( signed_block_v0_which == fc::get_index() ); + // this implementation is to avoid copy of signed_block_v0 to net_message + // matches which of net_message for signed_block_v0 + fc_dlog( logger, "sending v0 block ${bn}", ("bn", sb_v0.block_num()) ); + return buffer_factory::create_send_buffer( signed_block_v0_which, sb_v0 ); + } + }; + + struct trx_buffer_factory : public buffer_factory { + + /// caches result for subsequent calls, only provide same packed_transaction_ptr instance for each invocation. + /// protocol_version can differ per invocation as buffer_factory potentially caches multiple send buffers. + const send_buffer_type& get_send_buffer( const packed_transaction_ptr& trx, uint16_t protocol_version ) { + if( protocol_version >= proto_pruned_types ) { + if( !send_buffer ) { + send_buffer = create_send_buffer( trx ); + } + return send_buffer; + } else { + if( !send_buffer_v0 ) { + const auto v0 = trx->to_packed_transaction_v0(); + if( !v0 ) return send_buffer_v0; + send_buffer_v0 = create_send_buffer( *v0 ); + } + return send_buffer_v0; + } + } + + private: + send_buffer_type send_buffer_v0; + + private: + + static std::shared_ptr> create_send_buffer( const packed_transaction_ptr& trx ) { + static_assert( trx_message_v1_which == fc::get_index() ); + std::optional trx_id; + if( trx->get_estimated_size() > 1024 ) { // simple guess on threshold + fc_dlog( logger, "including trx id, est size: ${es}", ("es", trx->get_estimated_size()) ); + trx_id = trx->id(); + } + // const cast required, trx_message_v1 has non-const shared_ptr because FC_REFLECT does not work with const types + trx_message_v1 v1{std::move( trx_id ), std::const_pointer_cast( trx )}; + return buffer_factory::create_send_buffer( trx_message_v1_which, v1 ); + } + + static std::shared_ptr> create_send_buffer( const packed_transaction_v0& trx ) { + static_assert( packed_transaction_v0_which == fc::get_index() ); + // this implementation is to avoid copy of packed_transaction_v0 to net_message + // matches which of net_message for packed_transaction_v0 + return buffer_factory::create_send_buffer( packed_transaction_v0_which, trx ); + } + }; + + //------------------------------------------------------------------------ + + void connection::enqueue( const net_message& m ) { + verify_strand_in_this_thread( strand, __func__, __LINE__ ); + go_away_reason close_after_send = no_reason; + if (std::holds_alternative(m)) { + close_after_send = std::get(m).reason; + } + + buffer_factory buff_factory; + auto send_buffer = buff_factory.get_send_buffer( m ); + enqueue_buffer( send_buffer, close_after_send ); } - void connection::enqueue_block( const signed_block_ptr& sb, bool to_sync_queue) { - fc_dlog( logger, "enqueue block ${num}", ("num", sb->block_num()) ); + void connection::enqueue_block( const signed_block_ptr& b, bool to_sync_queue) { + fc_dlog( logger, "enqueue block ${num}", ("num", b->block_num()) ); verify_strand_in_this_thread( strand, __func__, __LINE__ ); - enqueue_buffer( create_send_buffer( sb ), no_reason, to_sync_queue); + + block_buffer_factory buff_factory; + auto sb = buff_factory.get_send_buffer( b, protocol_version.load() ); + if( !sb ) { + peer_wlog( this, "Sending go away for incomplete block #${n} ${id}...", + ("n", b->block_num())("id", b->calculate_id().str().substr(8,16)) ); + // unable to convert to v0 signed block and client doesn't support proto_pruned_types, so tell it to go away + enqueue( go_away_message( fatal_other ) ); + return; + } + enqueue_buffer( sb, no_reason, to_sync_queue); } void connection::enqueue_buffer( const std::shared_ptr>& send_buffer, @@ -1669,10 +1830,6 @@ namespace eosio { ("ep", c->peer_name())("lib", msg.last_irreversible_block_num)("head", msg.head_num) ("id", msg.head_id.str().substr(8,16)) ); c->syncing = false; - // wait for receipt of a notice message before initiating sync - if (c->protocol_version < proto_explicit_sync) { - start_sync( c, peer_lib ); - } return; } if (lib_num > msg.head_num ) { @@ -1922,8 +2079,8 @@ namespace eosio { bool dispatch_manager::have_block( const block_id_type& blkid ) const { std::lock_guard g(blk_state_mtx); - // by_block_id sorts have_block by greater so have_block == true will be the first one found - const auto& index = blk_state.get(); + // by_peer_block_id sorts have_block by greater so have_block == true will be the first one found + const auto& index = blk_state.get(); auto blk_itr = index.find( blkid ); if( blk_itr != index.end() ) { return blk_itr->have_block; @@ -1941,13 +2098,28 @@ namespace eosio { return added; } + // only adds if tid already exists, returns have_txn( tid ) + bool dispatch_manager::add_peer_txn( const transaction_id_type& tid, uint32_t connection_id ) { + std::lock_guard g( local_txns_mtx ); + auto tptr = local_txns.get().find( tid ); + if( tptr == local_txns.end() ) return false; + const auto expiration = tptr->expires; + + tptr = local_txns.get().find( std::make_tuple( std::ref( tid ), connection_id ) ); + if( tptr == local_txns.end() ) { + local_txns.insert( node_transaction_state{tid, expiration, 0, connection_id} ); + } + return true; + } + + // thread safe void dispatch_manager::update_txns_block_num( const signed_block_ptr& sb ) { update_block_num ubn( sb->block_num() ); std::lock_guard g( local_txns_mtx ); for( const auto& recpt : sb->transactions ) { - const transaction_id_type& id = (recpt.trx.which() == 0) ? recpt.trx.get() - : recpt.trx.get().id(); + const transaction_id_type& id = (recpt.trx.index() == 0) ? std::get(recpt.trx) + : std::get(recpt.trx).id(); auto range = local_txns.get().equal_range( id ); for( auto itr = range.first; itr != range.second; ++itr ) { local_txns.modify( itr, ubn ); @@ -2008,27 +2180,23 @@ namespace eosio { fc_dlog( logger, "bcast block ${b}", ("b", b->block_num()) ); if( my_impl->sync_master->syncing_with_peer() ) return; - - bool have_connection = false; - for_each_block_connection( [&have_connection]( auto& cp ) { + + block_buffer_factory buff_factory; + const auto bnum = b->block_num(); + for_each_block_connection( [this, &id, &bnum, &b, &buff_factory]( auto& cp ) { peer_dlog( cp, "socket_is_open ${s}, connecting ${c}, syncing ${ss}", ("s", cp->socket_is_open())("c", cp->connecting.load())("ss", cp->syncing.load()) ); - - if( !cp->current() ) { + if( !cp->current() ) return true; + send_buffer_type sb = buff_factory.get_send_buffer( b, cp->protocol_version.load() ); + if( !sb ) { + peer_wlog( cp, "Sending go away for incomplete block #${n} ${id}...", + ("n", b->block_num())("id", b->calculate_id().str().substr(8,16)) ); + // unable to convert to v0 signed block and client doesn't support proto_pruned_types, so tell it to go away + cp->enqueue( go_away_message( fatal_other ) ); return true; } - have_connection = true; - return false; - } ); - if( !have_connection ) return; - std::shared_ptr> send_buffer = create_send_buffer( b ); - - for_each_block_connection( [this, &id, bnum = b->block_num(), &send_buffer]( auto& cp ) { - if( !cp->current() ) { - return true; - } - cp->strand.post( [this, cp, id, bnum, send_buffer]() { + cp->strand.post( [this, cp, id, bnum, sb{std::move(sb)}]() { std::unique_lock g_conn( cp->conn_mtx ); bool has_block = cp->last_handshake_recv.last_irreversible_block_num >= bnum; g_conn.unlock(); @@ -2038,7 +2206,7 @@ namespace eosio { return; } fc_dlog( logger, "bcast block ${b} to ${p}", ("b", bnum)("p", cp->peer_name()) ); - cp->enqueue_buffer( send_buffer, no_reason ); + cp->enqueue_buffer( sb, no_reason ); } }); return true; @@ -2066,13 +2234,13 @@ namespace eosio { fc_dlog( logger, "rejected block ${id}", ("id", id) ); } - void dispatch_manager::bcast_transaction(const packed_transaction& trx) { - const auto& id = trx.id(); - time_point_sec trx_expiration = trx.expiration(); + void dispatch_manager::bcast_transaction(const packed_transaction_ptr& trx) { + const auto& id = trx->id(); + time_point_sec trx_expiration = trx->expiration(); node_transaction_state nts = {id, trx_expiration, 0, 0}; - std::shared_ptr> send_buffer; - for_each_connection( [this, &trx, &nts, &send_buffer]( auto& cp ) { + trx_buffer_factory buff_factory; + for_each_connection( [this, &trx, &nts, &buff_factory]( auto& cp ) { if( cp->is_blocks_only_connection() || !cp->current() ) { return true; } @@ -2080,13 +2248,12 @@ namespace eosio { if( !add_peer_txn(nts) ) { return true; } - if( !send_buffer ) { - send_buffer = create_send_buffer( trx ); - } - cp->strand.post( [cp, send_buffer]() { + send_buffer_type sb = buff_factory.get_send_buffer( trx, cp->protocol_version.load() ); + if( !sb ) return true; + cp->strand.post( [cp, sb{std::move(sb)}]() { fc_dlog( logger, "sending trx to ${n}", ("n", cp->peer_name()) ); - cp->enqueue_buffer( send_buffer, no_reason ); + cp->enqueue_buffer( sb, no_reason ); } ); return true; } ); @@ -2208,12 +2375,11 @@ namespace eosio { string port = c->peer_address().substr( colon + 1, colon2 == string::npos ? string::npos : colon2 - (colon + 1)); idump((host)(port)); c->set_connection_type( c->peer_address() ); - tcp::resolver::query query( tcp::v4(), host, port ); - // Note: need to add support for IPv6 too auto resolver = std::make_shared( my_impl->thread_pool->get_executor() ); connection_wptr weak_conn = c; - resolver->async_resolve( query, boost::asio::bind_executor( c->strand, + // Note: need to add support for IPv6 too + resolver->async_resolve( tcp::v4(), host, port, boost::asio::bind_executor( c->strand, [resolver, weak_conn]( const boost::system::error_code& err, tcp::resolver::results_type endpoints ) { auto c = weak_conn.lock(); if( !c ) return; @@ -2286,6 +2452,7 @@ namespace eosio { } ); if( from_addr < max_nodes_per_host && (max_client_count == 0 || visitors < max_client_count)) { fc_ilog( logger, "Accepted new connection: " + paddr_str ); + new_connection->set_heartbeat_timeout( heartbeat_timeout ); if( new_connection->start_session()) { std::lock_guard g_unique( connections_mtx ); connections.insert( new_connection ); @@ -2419,15 +2586,24 @@ namespace eosio { } close_connection = true; } + } + catch ( const std::bad_alloc& ) + { + throw; + } + catch ( const boost::interprocess::bad_alloc& ) + { + throw; + } + catch(const fc::exception &ex) + { + fc_elog( logger, "Exception in handling read data ${s}", ("s",ex.to_string()) ); + close_connection = true; } catch(const std::exception &ex) { fc_elog( logger, "Exception in handling read data: ${s}", ("s",ex.what()) ); close_connection = true; } - catch(const fc::exception &ex) { - fc_elog( logger, "Exception in handling read data ${s}", ("s",ex.to_string()) ); - close_connection = true; - } catch (...) { fc_elog( logger, "Undefined exception handling read data" ); close_connection = true; @@ -2447,102 +2623,192 @@ namespace eosio { // called from connection strand bool connection::process_next_message( uint32_t message_length ) { try { + latest_msg_time = get_time(); + // if next message is a block we already have, exit early auto peek_ds = pending_message_buffer.create_peek_datastream(); unsigned_int which{}; fc::raw::unpack( peek_ds, which ); - if( which == signed_block_which ) { - block_header bh; - fc::raw::unpack( peek_ds, bh ); - - const block_id_type blk_id = bh.id(); - const uint32_t blk_num = bh.block_num(); - if( my_impl->dispatcher->have_block( blk_id ) ) { - fc_dlog( logger, "canceling wait on ${p}, already received block ${num}, id ${id}...", - ("p", peer_name())("num", blk_num)("id", blk_id.str().substr(8,16)) ); - my_impl->sync_master->sync_recv_block( shared_from_this(), blk_id, blk_num, false ); - cancel_wait(); - - pending_message_buffer.advance_read_ptr( message_length ); - return true; - } - fc_dlog( logger, "${p} received block ${num}, id ${id}..., latency: ${latency}", - ("p", peer_name())("num", bh.block_num())("id", blk_id.str().substr(8,16)) - ("latency", (fc::time_point::now() - bh.timestamp).count()/1000) ); - if( !my_impl->sync_master->syncing_with_peer() ) { // guard against peer thinking it needs to send us old blocks - uint32_t lib = 0; - std::tie( lib, std::ignore, std::ignore, std::ignore, std::ignore, std::ignore ) = my_impl->get_chain_info(); - if( blk_num < lib ) { - std::unique_lock g( conn_mtx ); - const auto last_sent_lib = last_handshake_sent.last_irreversible_block_num; - g.unlock(); - if( blk_num < last_sent_lib ) { - fc_ilog( logger, "received block ${n} less than sent lib ${lib}", ("n", blk_num)("lib", last_sent_lib) ); - close(); - } else { - fc_ilog( logger, "received block ${n} less than lib ${lib}", ("n", blk_num)("lib", lib) ); - enqueue( (sync_request_message) {0, 0} ); - send_handshake(); - cancel_wait(); - } + if( which == signed_block_which || which == signed_block_v0_which ) { + return process_next_block_message( message_length ); - pending_message_buffer.advance_read_ptr( message_length ); - return true; - } - } + } else if( which == trx_message_v1_which || which == packed_transaction_v0_which ) { + return process_next_trx_message( message_length ); + } else { auto ds = pending_message_buffer.create_datastream(); - fc::raw::unpack( ds, which ); // throw away - shared_ptr ptr = std::make_shared(); - fc::raw::unpack( ds, *ptr ); - - auto is_webauthn_sig = []( const fc::crypto::signature& s ) { - return s.which() == fc::crypto::signature::storage_type::position(); - }; - bool has_webauthn_sig = is_webauthn_sig( ptr->producer_signature ); + net_message msg; + fc::raw::unpack( ds, msg ); + msg_handler m( shared_from_this() ); + std::visit( m, msg ); + } - constexpr auto additional_sigs_eid = additional_block_signatures_extension::extension_id(); - auto exts = ptr->validate_and_extract_extensions(); - if( exts.count( additional_sigs_eid ) ) { - const auto &additional_sigs = exts.lower_bound( additional_sigs_eid )->second.get().signatures; - has_webauthn_sig |= std::any_of( additional_sigs.begin(), additional_sigs.end(), is_webauthn_sig ); - } + } catch( const fc::exception& e ) { + fc_elog( logger, "Exception in handling message from ${p}: ${s}", + ("p", peer_name())("s", e.to_detail_string()) ); + close(); + return false; + } + return true; + } - if( has_webauthn_sig ) { - fc_dlog( logger, "WebAuthn signed block received from ${p}, closing connection", ("p", peer_name())); + // called from connection strand + bool connection::process_next_block_message(uint32_t message_length) { + auto peek_ds = pending_message_buffer.create_peek_datastream(); + unsigned_int which{}; + fc::raw::unpack( peek_ds, which ); // throw away + block_header bh; + fc::raw::unpack( peek_ds, bh ); + + const block_id_type blk_id = bh.calculate_id(); + const uint32_t blk_num = bh.block_num(); + if( my_impl->dispatcher->have_block( blk_id ) ) { + fc_dlog( logger, "canceling wait on ${p}, already received block ${num}, id ${id}...", + ("p", peer_name())("num", blk_num)("id", blk_id.str().substr(8,16)) ); + my_impl->sync_master->sync_recv_block( shared_from_this(), blk_id, blk_num, false ); + cancel_wait(); + + pending_message_buffer.advance_read_ptr( message_length ); + return true; + } + fc_dlog( logger, "${p} received block ${num}, id ${id}..., latency: ${latency}", + ("p", peer_name())("num", bh.block_num())("id", blk_id.str().substr(8,16)) + ("latency", (fc::time_point::now() - bh.timestamp).count()/1000) ); + if( !my_impl->sync_master->syncing_with_peer() ) { // guard against peer thinking it needs to send us old blocks + uint32_t lib = 0; + std::tie( lib, std::ignore, std::ignore, std::ignore, std::ignore, std::ignore ) = my_impl->get_chain_info(); + if( blk_num < lib ) { + std::unique_lock g( conn_mtx ); + const auto last_sent_lib = last_handshake_sent.last_irreversible_block_num; + g.unlock(); + if( blk_num < last_sent_lib ) { + fc_ilog( logger, "received block ${n} less than sent lib ${lib}", ("n", blk_num)("lib", last_sent_lib) ); close(); - return false; + } else { + fc_ilog( logger, "received block ${n} less than lib ${lib}", ("n", blk_num)("lib", lib) ); + enqueue( (sync_request_message) {0, 0} ); + send_handshake(); + cancel_wait(); } - handle_message( blk_id, std::move( ptr ) ); + pending_message_buffer.advance_read_ptr( message_length ); + return true; + } + } + + auto ds = pending_message_buffer.create_datastream(); + fc::raw::unpack( ds, which ); + shared_ptr ptr; + if( which == signed_block_which ) { + ptr = std::make_shared(); + fc::raw::unpack( ds, *ptr ); + } else { + signed_block_v0 sb_v0; + fc::raw::unpack( ds, sb_v0 ); + ptr = std::make_shared( std::move( sb_v0 ), true ); + } + + auto is_webauthn_sig = []( const fc::crypto::signature& s ) { + return s.which() == fc::get_index(); + }; + bool has_webauthn_sig = is_webauthn_sig( ptr->producer_signature ); + + constexpr auto additional_sigs_eid = additional_block_signatures_extension::extension_id(); + auto exts = ptr->validate_and_extract_extensions(); + if( exts.count( additional_sigs_eid ) ) { + const auto &additional_sigs = std::get(exts.lower_bound( additional_sigs_eid )->second).signatures; + has_webauthn_sig |= std::any_of( additional_sigs.begin(), additional_sigs.end(), is_webauthn_sig ); + } + + if( has_webauthn_sig ) { + fc_dlog( logger, "WebAuthn signed block received from ${p}, closing connection", ("p", peer_name())); + close(); + return false; + } - } else if( which == packed_transaction_which ) { - if( !my_impl->p2p_accept_transactions ) { - fc_dlog( logger, "p2p-accept-transaction=false - dropping txn" ); - pending_message_buffer.advance_read_ptr( message_length ); + handle_message( blk_id, std::move( ptr ) ); + return true; + } + + // called from connection strand + bool connection::process_next_trx_message(uint32_t message_length) { + if( !my_impl->p2p_accept_transactions ) { + fc_dlog( logger, "p2p-accept-transaction=false - dropping txn" ); + pending_message_buffer.advance_read_ptr( message_length ); + return true; + } + + const unsigned long trx_in_progress_sz = this->trx_in_progress_size.load(); + + auto report_dropping_trx = [](const transaction_id_type& trx_id, unsigned long trx_in_progress_sz) { + char reason[72]; + snprintf(reason, 72, "Dropping trx, too many trx in progress %lu bytes", trx_in_progress_sz); + my_impl->producer_plug->log_failed_transaction(trx_id, reason); + }; + + bool have_trx = false; + shared_ptr ptr; + auto ds = pending_message_buffer.create_datastream(); + const auto buff_size_start = pending_message_buffer.bytes_to_read(); + unsigned_int which{}; + fc::raw::unpack( ds, which ); + if( which == trx_message_v1_which ) { + std::optional trx_id; + fc::raw::unpack( ds, trx_id ); + if( trx_id ) { + if (trx_in_progress_sz > def_max_trx_in_progress_size) { + report_dropping_trx(*trx_id, trx_in_progress_sz); return true; } + have_trx = my_impl->dispatcher->add_peer_txn( *trx_id, connection_id ); + } - auto ds = pending_message_buffer.create_datastream(); - fc::raw::unpack( ds, which ); // throw away - shared_ptr ptr = std::make_shared(); - fc::raw::unpack( ds, *ptr ); - handle_message( std::move( ptr ) ); - + if( have_trx ) { + const auto buff_size_current = pending_message_buffer.bytes_to_read(); + pending_message_buffer.advance_read_ptr( message_length - (buff_size_start - buff_size_current) ); } else { - auto ds = pending_message_buffer.create_datastream(); - net_message msg; - fc::raw::unpack( ds, msg ); - msg_handler m( shared_from_this() ); - msg.visit( m ); + std::shared_ptr trx; + fc::raw::unpack( ds, trx ); + ptr = std::move( trx ); + + if (ptr && trx_id && *trx_id != ptr->id()) { + my_impl->producer_plug->log_failed_transaction(*trx_id, "Provided trx_id does not match provided packed_transaction"); + EOS_ASSERT(false, transaction_id_type_exception, + "Provided trx_id does not match provided packed_transaction" ); + } + + if( !trx_id ) { + if (trx_in_progress_sz > def_max_trx_in_progress_size) { + report_dropping_trx(ptr->id(), trx_in_progress_sz); + return true; + } + have_trx = my_impl->dispatcher->have_txn( ptr->id() ); + } + node_transaction_state nts = {ptr->id(), ptr->expiration(), 0, connection_id}; + my_impl->dispatcher->add_peer_txn( nts ); } - } catch( const fc::exception& e ) { - fc_elog( logger, "Exception in handling message from ${p}: ${s}", - ("p", peer_name())("s", e.to_detail_string()) ); - close(); - return false; + } else { + packed_transaction_v0 pt_v0; + fc::raw::unpack( ds, pt_v0 ); + if( trx_in_progress_sz > def_max_trx_in_progress_size) { + report_dropping_trx(pt_v0.id(), trx_in_progress_sz); + return true; + } + have_trx = my_impl->dispatcher->have_txn( pt_v0.id() ); + node_transaction_state nts = {pt_v0.id(), pt_v0.expiration(), 0, connection_id}; + my_impl->dispatcher->add_peer_txn( nts ); + if ( !have_trx ) { + ptr = std::make_shared( pt_v0, true ); + } + } + + if( have_trx ) { + fc_dlog( logger, "got a duplicate transaction - dropping" ); + return true; } + + handle_message( std::move( ptr ) ); return true; } @@ -2641,18 +2907,26 @@ namespace eosio { if(check.get() == this) continue; if(check->connected() && check->peer_name() == msg.p2p_address) { - // It's possible that both peers could arrive here at relatively the same time, so - // we need to avoid the case where they would both tell a different connection to go away. - // Using the sum of the initial handshake times of the two connections, we will - // arbitrarily (but consistently between the two peers) keep one of them. - std::unique_lock g_check_conn( check->conn_mtx ); - auto check_time = check->last_handshake_sent.time + check->last_handshake_recv.time; - g_check_conn.unlock(); - g_conn.lock(); - auto c_time = last_handshake_sent.time; - g_conn.unlock(); - if (msg.time + c_time <= check_time) - continue; + if (net_version < dup_goaway_resolution || msg.network_version < dup_goaway_resolution) { + // It's possible that both peers could arrive here at relatively the same time, so + // we need to avoid the case where they would both tell a different connection to go away. + // Using the sum of the initial handshake times of the two connections, we will + // arbitrarily (but consistently between the two peers) keep one of them. + + std::unique_lock g_check_conn( check->conn_mtx ); + auto check_time = check->last_handshake_sent.time + check->last_handshake_recv.time; + g_check_conn.unlock(); + g_conn.lock(); + auto c_time = last_handshake_sent.time; + g_conn.unlock(); + if (msg.time + c_time <= check_time) + continue; + } + else if (my_impl->p2p_address < msg.p2p_address) { + // only the connection from lower p2p_address to higher p2p_address will be considered as a duplicate, + // so there is no chance for both connections to be closed + continue; + } g_cnts.unlock(); fc_dlog( logger, "sending go_away duplicate to ${ep}", ("ep",msg.p2p_address) ); @@ -2679,7 +2953,7 @@ namespace eosio { protocol_version = my_impl->to_protocol_version(msg.network_version); if( protocol_version != net_version ) { fc_ilog( logger, "Local network version: ${nv} Remote version: ${mnv}", - ("nv", net_version)( "mnv", protocol_version ) ); + ("nv", net_version)("mnv", protocol_version.load()) ); } g_conn.lock(); @@ -2739,6 +3013,7 @@ namespace eosio { void connection::handle_message( const go_away_message& msg ) { peer_wlog( this, "received go_away_message, reason = ${r}", ("r", reason_str( msg.reason )) ); + bool retry = no_retry == no_reason; // if no previous go away message no_retry = msg.reason; if( msg.reason == duplicate ) { @@ -2751,11 +3026,13 @@ namespace eosio { retry = false; } flush_queues(); + close( retry ); // reconnect if wrong_version } void connection::handle_message( const time_message& msg ) { peer_dlog( this, "received time_message" ); + /* We've already lost however many microseconds it took to dispatch * the message, but it can't be helped. */ @@ -2902,48 +3179,36 @@ namespace eosio { peer_requested.reset(); flush_queues(); } else { - peer_requested = peer_sync_state( msg.start_block, msg.end_block, msg.start_block-1); + if (peer_requested) { + // This happens when peer already requested some range and sync is still in progress + // It could be higher in case of peer requested head catchup and current request is lib catchup + // So to make sure peer will receive all requested blocks we assign end_block to highest value + peer_requested->end_block = std::max(msg.end_block, peer_requested->end_block); + } + else { + peer_requested = peer_sync_state( msg.start_block, msg.end_block, msg.start_block-1); + } enqueue_sync_block(); } } size_t calc_trx_size( const packed_transaction_ptr& trx ) { - // transaction is stored packed and unpacked, double packed_size and size of signed as an approximation of use - return (trx->get_packed_transaction().size() * 2 + sizeof(trx->get_signed_transaction())) * 2 + - trx->get_packed_context_free_data().size() * 4 + - trx->get_signatures().size() * sizeof(signature_type); + return trx->get_estimated_size(); } void connection::handle_message( packed_transaction_ptr trx ) { const auto& tid = trx->id(); peer_dlog( this, "received packed_transaction ${id}", ("id", tid) ); - uint32_t trx_in_progress_sz = this->trx_in_progress_size.load(); - if( trx_in_progress_sz > def_max_trx_in_progress_size ) { - char reason[72]; - snprintf(reason, 72, "Dropping trx, too many trx in progress %lu bytes", (unsigned long) trx_in_progress_sz); - my_impl->producer_plug->log_failed_transaction(tid, reason); - return; - } - - bool have_trx = my_impl->dispatcher->have_txn( tid ); - node_transaction_state nts = {tid, trx->expiration(), 0, connection_id}; - my_impl->dispatcher->add_peer_txn( nts ); - - if( have_trx ) { - fc_dlog( logger, "got a duplicate transaction - dropping ${id}", ("id", tid) ); - return; - } - trx_in_progress_size += calc_trx_size( trx ); app().post( priority::low, [trx{std::move(trx)}, weak = weak_from_this()]() { my_impl->chain_plug->accept_transaction( trx, - [weak, trx](const static_variant& result) mutable { + [weak, trx](const std::variant& result) mutable { // next (this lambda) called from application thread - if (result.contains()) { - fc_dlog( logger, "bad packed_transaction : ${m}", ("m", result.get()->what()) ); + if (std::holds_alternative(result)) { + fc_dlog( logger, "bad packed_transaction : ${m}", ("m", std::get(result)->what()) ); } else { - const transaction_trace_ptr& trace = result.get(); + const transaction_trace_ptr& trace = std::get(result); if( !trace->except ) { fc_dlog( logger, "chain accepted transaction, bcast ${id}", ("id", trace->id) ); } else { @@ -2961,6 +3226,14 @@ namespace eosio { // called from connection strand void connection::handle_message( const block_id_type& id, signed_block_ptr ptr ) { peer_dlog( this, "received signed_block ${id}", ("id", ptr->block_num() ) ); + if( my_impl->p2p_reject_incomplete_blocks ) { + if( ptr->prune_state == signed_block::prune_state_type::incomplete ) { + peer_wlog( this, "Sending go away for incomplete block #${n} ${id}...", + ("n", ptr->block_num())("id", id.str().substr(8,16)) ); + enqueue( go_away_message( fatal_other ) ); + return; + } + } app().post(priority::medium, [ptr{std::move(ptr)}, id, c = shared_from_this()]() mutable { c->process_signed_block( id, std::move( ptr ) ); }); @@ -3082,10 +3355,12 @@ namespace eosio { if( my->in_shutdown ) return; fc_wlog( logger, "Peer keepalive ticked sooner than expected: ${m}", ("m", ec.message()) ); } - for_each_connection( []( auto& c ) { + + tstamp current_time = connection::get_time(); + for_each_connection( [current_time]( auto& c ) { if( c->socket_is_open() ) { - c->strand.post( [c]() { - c->send_time(); + c->strand.post([c, current_time]() { + c->check_heartbeat(current_time); } ); } return true; @@ -3168,6 +3443,13 @@ namespace eosio { controller& cc = chain_plug->chain(); dispatcher->strand.post( [this, bs]() { fc_dlog( logger, "signaled accepted_block, blk num = ${num}, id = ${id}", ("num", bs->block_num)("id", bs->id) ); + + auto blk_trace = fc_create_trace_with_id( "Block", bs->id ); + auto blk_span = fc_create_span( blk_trace, "Accepted" ); + fc_add_tag( blk_span, "block_id", bs->id ); + fc_add_tag( blk_span, "block_num", bs->block_num ); + fc_add_tag( blk_span, "block_time", bs->block->timestamp.to_time_point() ); + dispatcher->bcast_block( bs->block, bs->id ); }); } @@ -3178,8 +3460,16 @@ namespace eosio { controller& cc = chain_plug->chain(); if( cc.is_trusted_producer(block->producer) ) { dispatcher->strand.post( [this, block]() { - auto id = block->id(); + auto id = block->calculate_id(); fc_dlog( logger, "signaled pre_accepted_block, blk num = ${num}, id = ${id}", ("num", block->block_num())("id", id) ); + + auto blk_trace = fc_create_trace_with_id("Block", id); + auto blk_span = fc_create_span(blk_trace, "PreAccepted"); + fc_add_tag(blk_span, "block_id", id); + fc_add_tag(blk_span, "block_num", block->block_num()); + fc_add_tag(blk_span, "block_time", block->timestamp.to_time_point()); + fc_add_tag(blk_span, "producer", block->producer.to_string()); + dispatcher->bcast_block( block, id ); }); } @@ -3203,7 +3493,7 @@ namespace eosio { dispatcher->rejected_transaction(results.second->packed_trx(), head_blk_num); } else { fc_dlog( logger, "signaled ACK, trx-id = ${id}", ("id", id) ); - dispatcher->bcast_transaction(*results.second->packed_trx()); + dispatcher->bcast_transaction(results.second->packed_trx()); } }); } @@ -3247,7 +3537,7 @@ namespace eosio { try { peer_key = crypto::public_key(msg.sig, msg.token, true); } - catch (fc::exception& /*e*/) { + catch (const std::exception& /*e*/) { fc_elog( logger, "Peer ${peer} sent a handshake with an unrecoverable key.", ("peer", msg.p2p_address) ); return false; } @@ -3348,7 +3638,8 @@ namespace eosio { " p2p.blk.eos.io:9876:blk\n") ( "p2p-max-nodes-per-host", bpo::value()->default_value(def_max_nodes_per_host), "Maximum number of client nodes from any single IP address") ( "p2p-accept-transactions", bpo::value()->default_value(true), "Allow transactions received over p2p network to be evaluated and relayed if valid.") - ( "agent-name", bpo::value()->default_value("\"EOS Test Agent\""), "The name supplied to identify this node amongst the peers.") + ( "p2p-reject-incomplete-blocks", bpo::value()->default_value(true), "Reject pruned signed_blocks even in light validation") + ( "agent-name", bpo::value()->default_value("EOS Test Agent"), "The name supplied to identify this node amongst the peers.") ( "allowed-connection", bpo::value>()->multitoken()->default_value({"any"}, "any"), "Can be 'any' or 'producers' or 'specified' or 'none'. If 'specified', peer-key must be specified at least once. If only 'producers', peer-key is not required. 'producers' and 'specified' may be combined.") ( "peer-key", bpo::value>()->composing()->multitoken(), "Optional public key of peer allowed to connect. May be used multiple times.") ( "peer-private-key", boost::program_options::value>()->composing()->multitoken(), @@ -3370,6 +3661,8 @@ namespace eosio { " _port \tremote port number of peer\n\n" " _lip \tlocal IP address connected to peer\n\n" " _lport \tlocal port number connected to peer\n\n") + ( "p2p-keepalive-interval-ms", bpo::value()->default_value(def_keepalive_interval), "peer heartbeat keepalive message interval in milliseconds") + ; } @@ -3392,8 +3685,16 @@ namespace eosio { my->max_client_count = options.at( "max-clients" ).as(); my->max_nodes_per_host = options.at( "p2p-max-nodes-per-host" ).as(); my->p2p_accept_transactions = options.at( "p2p-accept-transactions" ).as(); + my->p2p_reject_incomplete_blocks = options.at("p2p-reject-incomplete-blocks").as(); my->use_socket_read_watermark = options.at( "use-socket-read-watermark" ).as(); + my->keepalive_interval = std::chrono::milliseconds( options.at( "p2p-keepalive-interval-ms" ).as() ); + EOS_ASSERT( my->keepalive_interval.count() > 0, chain::plugin_config_exception, + "p2p-keepalive_interval-ms must be greater than 0" ); + + if( options.count( "p2p-keepalive-interval-ms" )) { + my->heartbeat_timeout = std::chrono::milliseconds( options.at( "p2p-keepalive-interval-ms" ).as() * 2 ); + } if( options.count( "p2p-listen-endpoint" ) && options.at("p2p-listen-endpoint").as().length()) { my->p2p_address = options.at( "p2p-listen-endpoint" ).as(); @@ -3498,11 +3799,9 @@ namespace eosio { if( my->p2p_address.size() > 0 ) { auto host = my->p2p_address.substr( 0, my->p2p_address.find( ':' )); auto port = my->p2p_address.substr( host.size() + 1, my->p2p_address.size()); - tcp::resolver::query query( tcp::v4(), host.c_str(), port.c_str()); - // Note: need to add support for IPv6 too? - tcp::resolver resolver( my->thread_pool->get_executor() ); - listen_endpoint = *resolver.resolve( query ); + // Note: need to add support for IPv6 too? + listen_endpoint = *resolver.resolve( tcp::v4(), host, port ); my->acceptor.reset( new tcp::acceptor( my_impl->thread_pool->get_executor() ) ); @@ -3634,6 +3933,7 @@ namespace eosio { fc_dlog( logger, "calling active connector: ${h}", ("h", host) ); if( c->resolve_and_connect() ) { fc_dlog( logger, "adding new connection to the list: ${c}", ("c", c->peer_name()) ); + c->set_heartbeat_timeout( my->heartbeat_timeout ); my->connections.insert( c ); } return "added connection"; @@ -3652,12 +3952,12 @@ namespace eosio { return "no known connection for host"; } - optional net_plugin::status( const string& host )const { + std::optional net_plugin::status( const string& host )const { std::shared_lock g( my->connections_mtx ); auto con = my->find_connection( host ); if( con ) return con->get_status(); - return optional(); + return std::optional(); } vector net_plugin::connections()const { diff --git a/plugins/producer_api_plugin/producer.swagger.yaml b/plugins/producer_api_plugin/producer.swagger.yaml index aea863a7989..f7844957417 100644 --- a/plugins/producer_api_plugin/producer.swagger.yaml +++ b/plugins/producer_api_plugin/producer.swagger.yaml @@ -196,7 +196,7 @@ paths: type: array description: List of account names stored in the greylist items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" /producer/add_greylist_accounts: post: @@ -219,7 +219,7 @@ paths: type: array description: List of account names to add items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" responses: "200": @@ -249,7 +249,7 @@ paths: type: array description: List of account names to remove items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" responses: "200": @@ -260,7 +260,7 @@ paths: type: array description: List of account names stored in the greylist items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" /producer/get_whitelist_blacklist: post: @@ -287,19 +287,19 @@ paths: actor_whitelist: type: array items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" actor_blacklist: type: array items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" contract_whitelist: type: array items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" contract_blacklist: type: array items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" action_blacklist: type: array items: @@ -307,12 +307,12 @@ paths: description: Array of two string values, the account name as the first and action name as the second items: allOf: - - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" - - $ref: "https://eosio.github.io/schemata/v2.0/oas/CppSignature.yaml" + - $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" + - $ref: "https://eosio.github.io/schemata/v2.1/oas/CppSignature.yaml" key_blacklist: type: array items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/KeyType.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/KeyType.yaml" /producer/set_whitelist_blacklist: post: @@ -335,19 +335,19 @@ paths: actor_whitelist: type: array items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" actor_blacklist: type: array items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" contract_whitelist: type: array items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" contract_blacklist: type: array items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" action_blacklist: type: array items: @@ -355,12 +355,12 @@ paths: description: Array of two string values, the account name as the first and action name as the second items: allOf: - - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml" - - $ref: "https://eosio.github.io/schemata/v2.0/oas/CppSignature.yaml" + - $ref: "https://eosio.github.io/schemata/v2.1/oas/Name.yaml" + - $ref: "https://eosio.github.io/schemata/v2.1/oas/CppSignature.yaml" key_blacklist: type: array items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/KeyType.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/KeyType.yaml" responses: "200": @@ -387,11 +387,20 @@ paths: type: object description: Defines the snapshot to be created properties: + head_block_id: + $ref: "https://eosio.github.io/schemata/v2.1/oas/Sha256.yaml" + head_block_num: + type: integer + description: Highest block number on the chain + head_block_time: + type: string + description: Highest block unix timestamp + version: + type: integer + description: Version number snapshot_name: type: string description: The name of the snapshot - head_block_id: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml" responses: @@ -425,9 +434,9 @@ paths: description: Defines the integrity hash information details properties: integrity_hash: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Sha256.yaml" head_block_id: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Sha256.yaml" /producer/schedule_protocol_feature_activations: post: @@ -450,7 +459,7 @@ paths: type: array description: List of protocol features to activate items: - $ref: "https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml" + $ref: "https://eosio.github.io/schemata/v2.1/oas/Sha256.yaml" responses: "200": diff --git a/plugins/producer_api_plugin/producer_api_plugin.cpp b/plugins/producer_api_plugin/producer_api_plugin.cpp index 755009cdff0..61ceea8ec6f 100644 --- a/plugins/producer_api_plugin/producer_api_plugin.cpp +++ b/plugins/producer_api_plugin/producer_api_plugin.cpp @@ -27,11 +27,10 @@ struct async_result_visitor : public fc::visitor { } }; -#define CALL(api_name, api_handle, call_name, INVOKE, http_response_code) \ +#define CALL_WITH_400(api_name, api_handle, call_name, INVOKE, http_response_code) \ {std::string("/v1/" #api_name "/" #call_name), \ [&api_handle](string, string body, url_response_callback cb) mutable { \ try { \ - if (body.empty()) body = "{}"; \ INVOKE \ cb(http_response_code, fc::variant(result)); \ } catch (...) { \ @@ -43,15 +42,15 @@ struct async_result_visitor : public fc::visitor { {std::string("/v1/" #api_name "/" #call_name), \ [&api_handle](string, string body, url_response_callback cb) mutable { \ if (body.empty()) body = "{}"; \ - auto next = [cb, body](const fc::static_variant& result){\ - if (result.contains()) {\ + auto next = [cb, body](const std::variant& result){\ + if (std::holds_alternative(result)) {\ try {\ - result.get()->dynamic_rethrow_exception();\ + std::get(result)->dynamic_rethrow_exception();\ } catch (...) {\ http_plugin::handle_exception(#api_name, #call_name, body, cb);\ }\ } else {\ - cb(http_response_code, result.visit(async_result_visitor()));\ + cb(http_response_code, std::visit(async_result_visitor(), result));\ }\ };\ INVOKE\ @@ -59,28 +58,27 @@ struct async_result_visitor : public fc::visitor { } #define INVOKE_R_R(api_handle, call_name, in_param) \ - auto result = api_handle.call_name(fc::json::from_string(body).as()); + auto params = parse_params(body);\ + auto result = api_handle.call_name(std::move(params)); -#define INVOKE_R_R_R_R(api_handle, call_name, in_param0, in_param1, in_param2) \ - const auto& vs = fc::json::json::from_string(body).as(); \ - auto result = api_handle.call_name(vs.at(0).as(), vs.at(1).as(), vs.at(2).as()); +#define INVOKE_R_R_II(api_handle, call_name, in_param) \ + auto params = parse_params(body);\ + auto result = api_handle.call_name(std::move(params)); #define INVOKE_R_V(api_handle, call_name) \ + body = parse_params(body); \ auto result = api_handle.call_name(); #define INVOKE_R_V_ASYNC(api_handle, call_name)\ api_handle.call_name(next); #define INVOKE_V_R(api_handle, call_name, in_param) \ - api_handle.call_name(fc::json::from_string(body).as()); \ - eosio::detail::producer_api_plugin_response result{"ok"}; - -#define INVOKE_V_R_R(api_handle, call_name, in_param0, in_param1) \ - const auto& vs = fc::json::json::from_string(body).as(); \ - api_handle.call_name(vs.at(0).as(), vs.at(1).as()); \ + auto params = parse_params(body);\ + api_handle.call_name(std::move(params)); \ eosio::detail::producer_api_plugin_response result{"ok"}; #define INVOKE_V_V(api_handle, call_name) \ + body = parse_params(body); \ api_handle.call_name(); \ eosio::detail::producer_api_plugin_response result{"ok"}; @@ -91,38 +89,38 @@ void producer_api_plugin::plugin_startup() { auto& producer = app().get_plugin(); app().get_plugin().add_api({ - CALL(producer, producer, pause, + CALL_WITH_400(producer, producer, pause, INVOKE_V_V(producer, pause), 201), - CALL(producer, producer, resume, + CALL_WITH_400(producer, producer, resume, INVOKE_V_V(producer, resume), 201), - CALL(producer, producer, paused, + CALL_WITH_400(producer, producer, paused, INVOKE_R_V(producer, paused), 201), - CALL(producer, producer, get_runtime_options, + CALL_WITH_400(producer, producer, get_runtime_options, INVOKE_R_V(producer, get_runtime_options), 201), - CALL(producer, producer, update_runtime_options, + CALL_WITH_400(producer, producer, update_runtime_options, INVOKE_V_R(producer, update_runtime_options, producer_plugin::runtime_options), 201), - CALL(producer, producer, add_greylist_accounts, + CALL_WITH_400(producer, producer, add_greylist_accounts, INVOKE_V_R(producer, add_greylist_accounts, producer_plugin::greylist_params), 201), - CALL(producer, producer, remove_greylist_accounts, + CALL_WITH_400(producer, producer, remove_greylist_accounts, INVOKE_V_R(producer, remove_greylist_accounts, producer_plugin::greylist_params), 201), - CALL(producer, producer, get_greylist, + CALL_WITH_400(producer, producer, get_greylist, INVOKE_R_V(producer, get_greylist), 201), - CALL(producer, producer, get_whitelist_blacklist, + CALL_WITH_400(producer, producer, get_whitelist_blacklist, INVOKE_R_V(producer, get_whitelist_blacklist), 201), - CALL(producer, producer, set_whitelist_blacklist, + CALL_WITH_400(producer, producer, set_whitelist_blacklist, INVOKE_V_R(producer, set_whitelist_blacklist, producer_plugin::whitelist_blacklist), 201), - CALL(producer, producer, get_integrity_hash, + CALL_WITH_400(producer, producer, get_integrity_hash, INVOKE_R_V(producer, get_integrity_hash), 201), CALL_ASYNC(producer, producer, create_snapshot, producer_plugin::snapshot_information, INVOKE_R_V_ASYNC(producer, create_snapshot), 201), - CALL(producer, producer, get_scheduled_protocol_feature_activations, + CALL_WITH_400(producer, producer, get_scheduled_protocol_feature_activations, INVOKE_R_V(producer, get_scheduled_protocol_feature_activations), 201), - CALL(producer, producer, schedule_protocol_feature_activations, + CALL_WITH_400(producer, producer, schedule_protocol_feature_activations, INVOKE_V_R(producer, schedule_protocol_feature_activations, producer_plugin::scheduled_protocol_feature_activations), 201), - CALL(producer, producer, get_supported_protocol_features, - INVOKE_R_R(producer, get_supported_protocol_features, + CALL_WITH_400(producer, producer, get_supported_protocol_features, + INVOKE_R_R_II(producer, get_supported_protocol_features, producer_plugin::get_supported_protocol_features_params), 201), - CALL(producer, producer, get_account_ram_corrections, + CALL_WITH_400(producer, producer, get_account_ram_corrections, INVOKE_R_R(producer, get_account_ram_corrections, producer_plugin::get_account_ram_corrections_params), 201), }, appbase::priority::medium_high); } @@ -146,10 +144,8 @@ void producer_api_plugin::plugin_initialize(const variables_map& options) { #undef INVOKE_R_R -#undef INVOKE_R_R_R_R #undef INVOKE_R_V #undef INVOKE_V_R -#undef INVOKE_V_R_R #undef INVOKE_V_V #undef CALL diff --git a/plugins/producer_plugin/CMakeLists.txt b/plugins/producer_plugin/CMakeLists.txt index 4e5c23efe58..2a6d51c3701 100644 --- a/plugins/producer_plugin/CMakeLists.txt +++ b/plugins/producer_plugin/CMakeLists.txt @@ -2,12 +2,12 @@ file(GLOB HEADERS "include/eosio/producer_plugin/*.hpp") add_library( producer_plugin producer_plugin.cpp + pending_snapshot.cpp ${HEADERS} ) -target_link_libraries( producer_plugin chain_plugin http_client_plugin appbase eosio_chain ) +target_link_libraries( producer_plugin chain_plugin signature_provider_plugin appbase eosio_chain ) target_include_directories( producer_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/../chain_interface/include" ) add_subdirectory( test ) - diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/pending_snapshot.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/pending_snapshot.hpp new file mode 100644 index 00000000000..d981f723dc6 --- /dev/null +++ b/plugins/producer_plugin/include/eosio/producer_plugin/pending_snapshot.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include + +namespace eosio { + +class pending_snapshot { +public: + using next_t = producer_plugin::next_function; + + pending_snapshot(const chain::block_id_type& block_id, next_t& next, std::string pending_path, std::string final_path, blockvault::block_vault_interface* bv) + : block_id(block_id) + , next(next) + , pending_path(pending_path) + , final_path(final_path) + , blockvault(bv) + {} + + uint32_t get_height() const { + return chain::block_header::num_from_id(block_id); + } + + static bfs::path get_final_path(const chain::block_id_type& block_id, const bfs::path& snapshots_dir) { + return snapshots_dir / fc::format_string("snapshot-${id}.bin", fc::mutable_variant_object()("id", block_id)); + } + + static bfs::path get_pending_path(const chain::block_id_type& block_id, const bfs::path& snapshots_dir) { + return snapshots_dir / fc::format_string(".pending-snapshot-${id}.bin", fc::mutable_variant_object()("id", block_id)); + } + + static bfs::path get_temp_path(const chain::block_id_type& block_id, const bfs::path& snapshots_dir) { + return snapshots_dir / fc::format_string(".incomplete-snapshot-${id}.bin", fc::mutable_variant_object()("id", block_id)); + } + + producer_plugin::snapshot_information finalize( const chain::controller& chain ) const; + + chain::block_id_type block_id; + next_t next; + std::string pending_path; + std::string final_path; + blockvault::block_vault_interface* blockvault; +}; + +} // namespace eosio diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp index d3abdb02733..7daf28d2198 100644 --- a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp +++ b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include @@ -11,26 +11,26 @@ using boost::signals2::signal; class producer_plugin : public appbase::plugin { public: - APPBASE_PLUGIN_REQUIRES((chain_plugin)(http_client_plugin)) + APPBASE_PLUGIN_REQUIRES((chain_plugin)(signature_provider_plugin)) struct runtime_options { - fc::optional max_transaction_time; - fc::optional max_irreversible_block_age; - fc::optional produce_time_offset_us; - fc::optional last_block_time_offset_us; - fc::optional max_scheduled_transaction_time_per_block_ms; - fc::optional subjective_cpu_leeway_us; - fc::optional incoming_defer_ratio; - fc::optional greylist_limit; + std::optional max_transaction_time; + std::optional max_irreversible_block_age; + std::optional produce_time_offset_us; + std::optional last_block_time_offset_us; + std::optional max_scheduled_transaction_time_per_block_ms; + std::optional subjective_cpu_leeway_us; + std::optional incoming_defer_ratio; + std::optional greylist_limit; }; struct whitelist_blacklist { - fc::optional< flat_set > actor_whitelist; - fc::optional< flat_set > actor_blacklist; - fc::optional< flat_set > contract_whitelist; - fc::optional< flat_set > contract_blacklist; - fc::optional< flat_set< std::pair > > action_blacklist; - fc::optional< flat_set > key_blacklist; + std::optional< flat_set > actor_whitelist; + std::optional< flat_set > actor_blacklist; + std::optional< flat_set > contract_whitelist; + std::optional< flat_set > contract_blacklist; + std::optional< flat_set< std::pair > > action_blacklist; + std::optional< flat_set > key_blacklist; }; struct greylist_params { @@ -44,6 +44,9 @@ class producer_plugin : public appbase::plugin { struct snapshot_information { chain::block_id_type head_block_id; + uint32_t head_block_num; + fc::time_point head_block_time; + uint32_t version; std::string snapshot_name; }; @@ -57,19 +60,19 @@ class producer_plugin : public appbase::plugin { }; struct get_account_ram_corrections_params { - optional lower_bound; - optional upper_bound; - uint32_t limit = 10; - bool reverse = false; + std::optional lower_bound; + std::optional upper_bound; + uint32_t limit = 10; + bool reverse = false; }; struct get_account_ram_corrections_result { - std::vector rows; - optional more; + std::vector rows; + std::optional more; }; template - using next_function = std::function&)>; + using next_function = std::function&)>; producer_plugin(); virtual ~producer_plugin(); @@ -81,7 +84,6 @@ class producer_plugin : public appbase::plugin { bool is_producer_key(const chain::public_key_type& key) const; chain::signature_type sign_compact(const chain::public_key_type& key, const fc::sha256& digest) const; - uint32_t get_subjective_bill( const account_name& first_auth, const fc::time_point& now ) const; virtual void plugin_initialize(const boost::program_options::variables_map& options); virtual void plugin_startup(); @@ -123,7 +125,7 @@ FC_REFLECT(eosio::producer_plugin::runtime_options, (max_transaction_time)(max_i FC_REFLECT(eosio::producer_plugin::greylist_params, (accounts)); FC_REFLECT(eosio::producer_plugin::whitelist_blacklist, (actor_whitelist)(actor_blacklist)(contract_whitelist)(contract_blacklist)(action_blacklist)(key_blacklist) ) FC_REFLECT(eosio::producer_plugin::integrity_hash_information, (head_block_id)(integrity_hash)) -FC_REFLECT(eosio::producer_plugin::snapshot_information, (head_block_id)(snapshot_name)) +FC_REFLECT(eosio::producer_plugin::snapshot_information, (head_block_id)(head_block_num)(head_block_time)(version)(snapshot_name)) FC_REFLECT(eosio::producer_plugin::scheduled_protocol_feature_activations, (protocol_features_to_activate)) FC_REFLECT(eosio::producer_plugin::get_supported_protocol_features_params, (exclude_disabled)(exclude_unactivatable)) FC_REFLECT(eosio::producer_plugin::get_account_ram_corrections_params, (lower_bound)(upper_bound)(limit)(reverse)) diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/subjective_billing.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/subjective_billing.hpp index 911a99a177e..3361575ce07 100644 --- a/plugins/producer_plugin/include/eosio/producer_plugin/subjective_billing.hpp +++ b/plugins/producer_plugin/include/eosio/producer_plugin/subjective_billing.hpp @@ -13,6 +13,8 @@ #include #include +#include + namespace eosio { namespace bmi = boost::multi_index; @@ -53,8 +55,8 @@ class subjective_billing { } }; - using account_subjective_bill_cache = std::map; - using block_subjective_bill_cache = std::map; + using account_subjective_bill_cache = std::unordered_map; + using block_subjective_bill_cache = std::unordered_map; bool _disabled = false; trx_cache_index _trx_cache_index; @@ -90,8 +92,8 @@ class subjective_billing { void remove_subjective_billing( const block_state_ptr& bsp, uint32_t time_ordinal ) { if( !_trx_cache_index.empty() ) { for( const auto& receipt : bsp->block->transactions ) { - if( receipt.trx.contains() ) { - const auto& pt = receipt.trx.get(); + if( std::holds_alternative(receipt.trx) ) { + const auto& pt = std::get(receipt.trx); remove_subjective_billing( pt.id(), time_ordinal ); } } @@ -113,6 +115,7 @@ class subjective_billing { public: void disable() { _disabled = true; } + bool is_disabled() const { return _disabled; } void disable_account( chain::account_name a ) { _disabled_accounts.emplace( a ); } /// @param in_pending_block pass true if pt's bill time is accounted for in the pending block diff --git a/plugins/producer_plugin/pending_snapshot.cpp b/plugins/producer_plugin/pending_snapshot.cpp new file mode 100644 index 00000000000..772956c37f5 --- /dev/null +++ b/plugins/producer_plugin/pending_snapshot.cpp @@ -0,0 +1,32 @@ +#include +#include + +namespace eosio { + +producer_plugin::snapshot_information pending_snapshot::finalize( const chain::controller& chain ) const { + auto block_ptr = chain.fetch_block_by_id( block_id ); + auto in_chain = (bool)block_ptr; + boost::system::error_code ec; + + if (!in_chain) { + bfs::remove(bfs::path(pending_path), ec); + EOS_THROW(chain::snapshot_finalization_exception, + "Snapshotted block was forked out of the chain. ID: ${block_id}", + ("block_id", block_id)); + } + + bfs::rename(bfs::path(pending_path), bfs::path(final_path), ec); + EOS_ASSERT(!ec, chain::snapshot_finalization_exception, + "Unable to finalize valid snapshot of block number ${bn}: [code: ${ec}] ${message}", + ("bn", get_height()) + ("ec", ec.value()) + ("message", ec.message())); + + if (blockvault) { + blockvault->propose_snapshot( blockvault::watermark_t{block_ptr->block_num(), block_ptr->timestamp}, final_path.c_str() ); + } + + return {block_id, block_ptr->block_num(), block_ptr->timestamp, chain::chain_snapshot_header::current_version, final_path}; +} + +} // namespace eosio diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index 6da601f263e..09ba7c86fe8 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -7,10 +8,11 @@ #include #include #include +#include +#include #include #include -#include #include #include @@ -18,7 +20,6 @@ #include #include -#include #include #include #include @@ -38,7 +39,6 @@ using boost::multi_index_container; using std::string; using std::vector; -using std::deque; using boost::signals2::scoped_connection; #undef FC_LOG_AND_DROP @@ -68,7 +68,7 @@ using boost::signals2::scoped_connection; const fc::string logger_name("producer_plugin"); fc::logger _log; -const fc::string trx_successful_trace_logger_name("transaction_tracing"); +const fc::string trx_successful_trace_logger_name("transaction_success_tracing"); fc::logger _trx_successful_trace_log; const fc::string trx_failed_trace_logger_name("transaction_failure_tracing"); @@ -108,60 +108,6 @@ using transaction_id_with_expiry_index = multi_index_container< struct by_height; -class pending_snapshot { -public: - using next_t = producer_plugin::next_function; - - pending_snapshot(const block_id_type& block_id, next_t& next, std::string pending_path, std::string final_path) - : block_id(block_id) - , next(next) - , pending_path(pending_path) - , final_path(final_path) - {} - - uint32_t get_height() const { - return block_header::num_from_id(block_id); - } - - static bfs::path get_final_path(const block_id_type& block_id, const bfs::path& snapshots_dir) { - return snapshots_dir / fc::format_string("snapshot-${id}.bin", fc::mutable_variant_object()("id", block_id)); - } - - static bfs::path get_pending_path(const block_id_type& block_id, const bfs::path& snapshots_dir) { - return snapshots_dir / fc::format_string(".pending-snapshot-${id}.bin", fc::mutable_variant_object()("id", block_id)); - } - - static bfs::path get_temp_path(const block_id_type& block_id, const bfs::path& snapshots_dir) { - return snapshots_dir / fc::format_string(".incomplete-snapshot-${id}.bin", fc::mutable_variant_object()("id", block_id)); - } - - producer_plugin::snapshot_information finalize( const chain::controller& chain ) const { - auto in_chain = (bool)chain.fetch_block_by_id( block_id ); - boost::system::error_code ec; - - if (!in_chain) { - bfs::remove(bfs::path(pending_path), ec); - EOS_THROW(snapshot_finalization_exception, - "Snapshotted block was forked out of the chain. ID: ${block_id}", - ("block_id", block_id)); - } - - bfs::rename(bfs::path(pending_path), bfs::path(final_path), ec); - EOS_ASSERT(!ec, snapshot_finalization_exception, - "Unable to finalize valid snapshot of block number ${bn}: [code: ${ec}] ${message}", - ("bn", get_height()) - ("ec", ec.value()) - ("message", ec.message())); - - return {block_id, final_path}; - } - - block_id_type block_id; - next_t next; - std::string pending_path; - std::string final_path; -}; - using pending_snapshot_index = multi_index_container< pending_snapshot, indexed_by< @@ -175,22 +121,39 @@ enum class pending_block_mode { speculating }; +class producer_plugin_impl; +class block_only_sync : public blockvault::sync_callback { + producer_plugin_impl* _impl; + boost::asio::deadline_timer _start_sync_timer; + bool _pending = false; + + public: + block_only_sync(producer_plugin_impl* impl, boost::asio::io_service& io) + : _impl(impl), _start_sync_timer(io) {} + + bool is_pending() const { return _pending; } + void cancel() { _start_sync_timer.cancel(); } + void schedule(); + void on_snapshot(const char* snapshot_filename) override; + void on_block(eosio::chain::signed_block_ptr block) override; +}; class producer_plugin_impl : public std::enable_shared_from_this { public: producer_plugin_impl(boost::asio::io_service& io) :_timer(io) + ,_block_vault_resync(this, io) ,_transaction_ack_channel(app().get_channel()) { } - optional calculate_next_block_time(const account_name& producer_name, const block_timestamp_type& current_block_time) const; + std::optional calculate_next_block_time(const account_name& producer_name, const block_timestamp_type& current_block_time) const; void schedule_production_loop(); void schedule_maybe_produce_block( bool exhausted ); void produce_block(); bool maybe_produce_block(); + bool remove_expired_trxs( const fc::time_point& deadline ); bool block_is_exhausted() const; - bool remove_expired_persisted_trxs( const fc::time_point& deadline ); bool remove_expired_blacklisted_trxs( const fc::time_point& deadline ); bool process_unapplied_trxs( const fc::time_point& deadline ); void process_scheduled_and_incoming_trxs( const fc::time_point& deadline, size_t& pending_incoming_process_limit ); @@ -200,15 +163,16 @@ class producer_plugin_impl : public std::enable_shared_from_this; + using signature_provider_type = signature_provider_plugin::signature_provider_type; std::map _signature_providers; std::set _producers; boost::asio::deadline_timer _timer; + block_only_sync _block_vault_resync; using producer_watermark = std::pair; std::map _producer_watermarks; pending_block_mode _pending_block_mode = pending_block_mode::speculating; unapplied_transaction_queue _unapplied_transactions; - fc::optional _thread_pool; + std::optional _thread_pool; std::atomic _max_transaction_time_ms; // modified by app thread, read by net_plugin thread pool fc::microseconds _max_irreversible_block_age_us; @@ -221,28 +185,30 @@ class producer_plugin_impl : public std::enable_shared_from_this _protocol_features_to_activate; bool _protocol_features_signaled = false; // to mark whether it has been signaled in start_block - chain_plugin* chain_plug = nullptr; + chain_plugin* chain_plug = nullptr; + eosio::blockvault::block_vault_interface* blockvault = nullptr; + uint32_t _latest_rejected_block_num = 0; - incoming::channels::block::channel_type::handle _incoming_block_subscription; - incoming::channels::transaction::channel_type::handle _incoming_transaction_subscription; + incoming::channels::block::channel_type::handle _incoming_block_subscription; + incoming::channels::transaction::channel_type::handle _incoming_transaction_subscription; - compat::channels::transaction_ack::channel_type& _transaction_ack_channel; + compat::channels::transaction_ack::channel_type& _transaction_ack_channel; incoming::methods::block_sync::method_type::handle _incoming_block_sync_provider; + incoming::methods::blockvault_sync::method_type::handle _incoming_blockvault_sync_provider; incoming::methods::transaction_async::method_type::handle _incoming_transaction_async_provider; - transaction_id_with_expiry_index _blacklisted_transactions; - pending_snapshot_index _pending_snapshot_index; - subjective_billing _subjective_billing; + transaction_id_with_expiry_index _blacklisted_transactions; + pending_snapshot_index _pending_snapshot_index; + subjective_billing _subjective_billing; - fc::optional _accepted_block_connection; - fc::optional _accepted_block_header_connection; - fc::optional _irreversible_block_connection; + std::optional _accepted_block_connection; + std::optional _accepted_block_header_connection; + std::optional _irreversible_block_connection; /* * HACK ALERT @@ -281,8 +247,11 @@ class producer_plugin_impl : public std::enable_shared_from_thischain(); + + const auto& id = block->calculate_id(); + auto blk_num = block->block_num(); + + fc_dlog(_log, "syncing blockvault block ${n} ${id}", ("n", blk_num)("id", id)); + + if (check_connectivity) { + auto previous = chain.fetch_block_by_id(block->previous); + if (!previous) { + fc_dlog(_log, "Don't have previous block for block number ${bn}, looking for block id ${pbi}", + ("bn", block->block_num())("pbi", block->previous)); + return true; + } + } + + auto existing = chain.fetch_block_by_id( id ); + if( existing ) { return true; } + + // start processing of block + auto bsf = chain.create_block_state_future( id, block ); + + // abort the pending block + _unapplied_transactions.add_aborted( chain.abort_block() ); + + // push the new block + auto handle_error = [&](const auto& e) + { + fc_elog(_log, (e.to_detail_string())); + throw; + }; + + try { + block_state_ptr blk_state = chain.push_block( bsf, [this]( const branch_type& forked_branch ) { + _unapplied_transactions.add_forked( forked_branch ); + }, [this]( const transaction_id_type& id ) { + return _unapplied_transactions.get_trx( id ); + } ); + } catch ( const guard_exception& e ) { + chain_plugin::handle_guard_exception(e); + return false; + } catch ( const std::bad_alloc& ) { + chain_plugin::handle_bad_alloc(); + } catch ( boost::interprocess::bad_alloc& ) { + chain_plugin::handle_db_exhaustion(); + } catch( const fc::exception& e ) { + handle_error(e); + } catch (const std::exception& e) { + handle_error(fc::std_exception_wrapper::from_current_exception(e)); + } + + return true; + } + bool on_incoming_block(const signed_block_ptr& block, const std::optional& block_id) { auto& chain = chain_plug->chain(); - if ( _pending_block_mode == pending_block_mode::producing ) { + if ( _pending_block_mode == pending_block_mode::producing) { fc_wlog( _log, "dropped incoming block #${num} id: ${id}", ("num", block->block_num())("id", block_id ? (*block_id).str() : "UNKNOWN") ); return false; } - const auto& id = block_id ? *block_id : block->id(); + const auto& id = block_id ? *block_id : block->calculate_id(); auto blk_num = block->block_num(); fc_dlog(_log, "received incoming block ${n} ${id}", ("n", blk_num)("id", id)); @@ -337,7 +361,7 @@ class producer_plugin_impl : public std::enable_shared_from_this().publish( priority::medium, block ); + throw; + }; + try { - chain.push_block( bsf, [this]( const branch_type& forked_branch ) { + block_state_ptr blk_state = chain.push_block( bsf, [this]( const branch_type& forked_branch ) { _unapplied_transactions.add_forked( forked_branch ); }, [this]( const transaction_id_type& id ) { return _unapplied_transactions.get_trx( id ); } ); + + if ( blockvault != nullptr ) { + if (_block_vault_resync.is_pending() && _producers.count( block->producer ) > 0 ) { + // Cancel any pending resync from blockvault if we received any blocks from the same logical producer + _block_vault_resync.cancel(); + } + blockvault->async_append_external_block(blk_state->dpos_irreversible_blocknum, blk_state->block, [](bool){}); + } } catch ( const guard_exception& e ) { chain_plugin::handle_guard_exception(e); return false; - } catch( const fc::exception& e ) { - elog((e.to_detail_string())); - app().get_channel().publish( priority::medium, block ); - throw; } catch ( const std::bad_alloc& ) { chain_plugin::handle_bad_alloc(); } catch ( boost::interprocess::bad_alloc& ) { chain_plugin::handle_db_exhaustion(); + } catch ( const fork_database_exception& e ) { + elog("Cannot recover from ${e}. Shutting down.", ("e", e.to_detail_string())); + appbase::app().quit(); + } catch( const fc::exception& e ) { + handle_error(e); + } catch (const std::exception& e) { + handle_error(fc::std_exception_wrapper::from_current_exception(e)); } const auto& hbs = chain.head_block_state(); @@ -388,49 +430,13 @@ class producer_plugin_impl : public std::enable_shared_from_this>> _incoming_transactions; - - private: - static uint64_t calc_size( const transaction_metadata_ptr& trx ) { - return trx->packed_trx()->get_unprunable_size() + trx->packed_trx()->get_prunable_size() + sizeof( *trx ); - } - - void add_size( const transaction_metadata_ptr& trx ) { - auto size = calc_size( trx ); - EOS_ASSERT( size_in_bytes + size < max_incoming_transaction_queue_size, tx_resource_exhaustion, "Transaction exceeded producer resource limit" ); - size_in_bytes += size; - } - - public: - void set_max_incoming_transaction_queue_size( uint64_t v ) { max_incoming_transaction_queue_size = v; } - - void add( const transaction_metadata_ptr& trx, bool persist_until_expired, next_function next ) { - add_size( trx ); - _incoming_transactions.emplace_back( trx, persist_until_expired, std::move( next ) ); - } - - void add_front( const transaction_metadata_ptr& trx, bool persist_until_expired, next_function next ) { - add_size( trx ); - _incoming_transactions.emplace_front( trx, persist_until_expired, std::move( next ) ); - } - - auto pop_front() { - EOS_ASSERT( !_incoming_transactions.empty(), producer_exception, "logic error, front() called on empty incoming_transactions" ); - auto intrx = _incoming_transactions.front(); - _incoming_transactions.pop_front(); - const transaction_metadata_ptr& trx = std::get<0>( intrx ); - size_in_bytes -= calc_size( trx ); - return intrx; - } - - bool empty()const { return _incoming_transactions.empty(); } - size_t size()const { return _incoming_transactions.size(); } - }; + void restart_speculative_block() { + chain::controller& chain = chain_plug->chain(); + // abort the pending block + _unapplied_transactions.add_aborted( chain.abort_block() ); - incoming_transaction_queue _pending_incoming_transactions; + schedule_production_loop(); + } void on_incoming_transaction_async(const packed_transaction_ptr& trx, bool persist_until_expired, next_function next) { chain::controller& chain = chain_plug->chain(); @@ -446,8 +452,6 @@ class producer_plugin_impl : public std::enable_shared_from_thisid())("a",trx->get_transaction().first_authorizer())("why",ex->what())); fc_dlog(_trx_failed_trace_log, "[TRX_TRACE] Speculative execution is REJECTING tx: ${txid}, auth: ${a} : ${why} ", ("txid", trx->id())("a",trx->get_transaction().first_authorizer())("why",ex->what())); next(ex); @@ -457,6 +461,8 @@ class producer_plugin_impl : public std::enable_shared_from_thisprocess_incoming_transaction_async( result, persist_until_expired, next ) ) { if( self->_pending_block_mode == pending_block_mode::producing ) { self->schedule_maybe_produce_block( true ); + } else { + self->restart_speculative_block(); } } } CATCH_AND_CALL(exception_handler); @@ -469,32 +475,22 @@ class producer_plugin_impl : public std::enable_shared_from_thischain(); - auto send_response = [this, &trx, &chain, &next](const fc::static_variant& response) { + auto send_response = [this, &trx, &chain, &next](const std::variant& response) { next(response); - if (response.contains()) { - _transaction_ack_channel.publish(priority::low, std::pair(response.get(), trx)); + if (std::holds_alternative(response)) { + _transaction_ack_channel.publish(priority::low, std::pair(std::get(response), trx)); if (_pending_block_mode == pending_block_mode::producing) { - fc_dlog(_trx_successful_trace_log, "[TRX_TRACE] Block ${block_num} for producer ${prod} is REJECTING tx: ${txid}, auth: ${a} : ${why} ", - ("block_num", chain.head_block_num() + 1) - ("prod", get_pending_block_producer()) - ("txid", trx->id()) - ("a", trx->packed_trx()->get_transaction().first_authorizer()) - ("why",response.get()->what())); fc_dlog(_trx_failed_trace_log, "[TRX_TRACE] Block ${block_num} for producer ${prod} is REJECTING tx: ${txid}, auth: ${a} : ${why} ", ("block_num", chain.head_block_num() + 1) ("prod", get_pending_block_producer()) ("txid", trx->id()) ("a", trx->packed_trx()->get_transaction().first_authorizer()) - ("why",response.get()->what())); + ("why",std::get(response)->what())); } else { - fc_dlog(_trx_successful_trace_log, "[TRX_TRACE] Speculative execution is REJECTING tx: ${txid}, auth: ${a} : ${why} ", - ("txid", trx->id()) - ("a", trx->packed_trx()->get_transaction().first_authorizer()) - ("why",response.get()->what())); fc_dlog(_trx_failed_trace_log, "[TRX_TRACE] Speculative execution is REJECTING tx: ${txid}, auth: ${a} : ${why} ", ("txid", trx->id()) ("a", trx->packed_trx()->get_transaction().first_authorizer()) - ("why",response.get()->what())); + ("why",std::get(response)->what())); } } else { _transaction_ack_channel.publish(priority::low, std::pair(nullptr, trx)); @@ -532,7 +528,7 @@ class producer_plugin_impl : public std::enable_shared_from_thiselapsed)); if( trace->except ) { if( exception_is_exhausted( *trace->except, deadline_is_subjective )) { - _pending_incoming_transactions.add( trx, persist_until_expired, next ); + _unapplied_transactions.add_incoming( trx, persist_until_expired, next ); if( _pending_block_mode == pending_block_mode::producing ) { - fc_dlog(_log, "[TRX_TRACE] Block ${block_num} for producer ${prod} COULD NOT FIT, tx: ${txid} RETRYING ", + fc_dlog(_log, "[TRX_TRACE] Block ${block_num} for producer ${prod} COULD NOT FIT, tx: ${txid} RETRYING, ec: ${c} ", ("block_num", chain.head_block_num() + 1) ("prod", get_pending_block_producer()) - ("txid", trx->id())); + ("txid", trx->id())("c", trace->except->code())); } else { - fc_dlog(_log, "[TRX_TRACE] Speculative execution COULD NOT FIT tx: ${txid} RETRYING", - ("txid", trx->id())); + fc_dlog(_log, "[TRX_TRACE] Speculative execution COULD NOT FIT tx: ${txid} RETRYING, ec: ${c}", + ("txid", trx->id())("c", trace->except->code())); } exhausted = block_is_exhausted(); } else { @@ -634,8 +630,8 @@ class producer_plugin_impl : public std::enable_shared_from_this& weak_this, optional wake_up_time); - optional calculate_producer_wake_up_time( const block_timestamp_type& ref_block_time ) const; + void schedule_delayed_production_loop(const std::weak_ptr& weak_this, std::optional wake_up_time); + std::optional calculate_producer_wake_up_time( const block_timestamp_type& ref_block_time ) const; }; @@ -690,15 +686,7 @@ void producer_plugin::set_program_options( ("signature-provider", boost::program_options::value>()->composing()->multitoken()->default_value( {default_priv_key.get_public_key().to_string() + "=KEY:" + default_priv_key.to_string()}, default_priv_key.get_public_key().to_string() + "=KEY:" + default_priv_key.to_string()), - "Key=Value pairs in the form =\n" - "Where:\n" - " \tis a string form of a vaild EOSIO public key\n\n" - " \tis a string in the form :\n\n" - " \tis KEY, or KEOSD\n\n" - " KEY: \tis a string form of a valid EOSIO private key which maps to the provided public key\n\n" - " KEOSD: \tis the URL where keosd is available and the approptiate wallet(s) are unlocked") - ("keosd-provider-timeout", boost::program_options::value()->default_value(5), - "Limits the maximum time (in milliseconds) that is allowed for sending blocks to a keosd provider for signing") + app().get_plugin().signature_provider_help_text()) ("greylist-account", boost::program_options::value>()->composing()->multitoken(), "account that can not access to extended CPU/NET virtual resources") ("greylist-limit", boost::program_options::value()->default_value(1000), @@ -749,11 +737,6 @@ bool producer_plugin::is_producer_key(const chain::public_key_type& key) const return false; } -uint32_t producer_plugin::get_subjective_bill( const account_name& first_auth, const fc::time_point& now ) const -{ - return my->_subjective_billing.get_subjective_bill( first_auth, now ); -} - chain::signature_type producer_plugin::sign_compact(const chain::public_key_type& key, const fc::sha256& digest) const { if(key != chain::public_key_type()) { @@ -780,50 +763,17 @@ if( options.count(op_name) ) { \ } \ } -static producer_plugin_impl::signature_provider_type -make_key_signature_provider(const private_key_type& key) { - return [key]( const chain::digest_type& digest ) { - return key.sign(digest); - }; -} - -static producer_plugin_impl::signature_provider_type -make_keosd_signature_provider(const std::shared_ptr& impl, const string& url_str, const public_key_type pubkey) { - fc::url keosd_url; - if(boost::algorithm::starts_with(url_str, "unix://")) - //send the entire string after unix:// to http_plugin. It'll auto-detect which part - // is the unix socket path, and which part is the url to hit on the server - keosd_url = fc::url("unix", url_str.substr(7), ostring(), ostring(), ostring(), ostring(), ovariant_object(), fc::optional()); - else - keosd_url = fc::url(url_str); - std::weak_ptr weak_impl = impl; - - return [weak_impl, keosd_url, pubkey]( const chain::digest_type& digest ) { - auto impl = weak_impl.lock(); - if (impl) { - fc::variant params; - fc::to_variant(std::make_pair(digest, pubkey), params); - auto deadline = impl->_keosd_provider_timeout_us.count() >= 0 ? fc::time_point::now() + impl->_keosd_provider_timeout_us : fc::time_point::maximum(); - return app().get_plugin().get_client().post_sync(keosd_url, params, deadline).as(); - } else { - return signature_type(); - } - }; -} - void producer_plugin::plugin_initialize(const boost::program_options::variables_map& options) { try { + auto blockvault_plug = app().find_plugin(); + my->blockvault = blockvault_plug ? blockvault_plug->get() : nullptr; + my->chain_plug = app().find_plugin(); EOS_ASSERT( my->chain_plug, plugin_config_exception, "chain_plugin not found" ); my->_options = &options; LOAD_VALUE_SET(options, "producer-name", my->_producers) chain::controller& chain = my->chain_plug->chain(); - unapplied_transaction_queue::process_mode unapplied_mode = - (chain.get_read_mode() != chain::db_read_mode::SPECULATIVE) ? unapplied_transaction_queue::process_mode::non_speculative : - my->_producers.empty() ? unapplied_transaction_queue::process_mode::speculative_non_producer : - unapplied_transaction_queue::process_mode::speculative_producer; - my->_unapplied_transactions.set_mode( unapplied_mode ); if( options.count("private-key") ) { @@ -832,10 +782,10 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_ { try { auto key_id_to_wif_pair = dejsonify>(key_id_to_wif_pair_string); - my->_signature_providers[key_id_to_wif_pair.first] = make_key_signature_provider(key_id_to_wif_pair.second); + my->_signature_providers[key_id_to_wif_pair.first] = app().get_plugin().signature_provider_for_private_key(key_id_to_wif_pair.second); auto blanked_privkey = std::string(key_id_to_wif_pair.second.to_string().size(), '*' ); wlog("\"private-key\" is DEPRECATED, use \"signature-provider=${pub}=KEY:${priv}\"", ("pub",key_id_to_wif_pair.first)("priv", blanked_privkey)); - } catch ( fc::exception& e ) { + } catch ( const std::exception& e ) { elog("Malformed private key pair"); } } @@ -845,32 +795,16 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_ const std::vector key_spec_pairs = options["signature-provider"].as>(); for (const auto& key_spec_pair : key_spec_pairs) { try { - auto delim = key_spec_pair.find("="); - EOS_ASSERT(delim != std::string::npos, plugin_config_exception, "Missing \"=\" in the key spec pair"); - auto pub_key_str = key_spec_pair.substr(0, delim); - auto spec_str = key_spec_pair.substr(delim + 1); - - auto spec_delim = spec_str.find(":"); - EOS_ASSERT(spec_delim != std::string::npos, plugin_config_exception, "Missing \":\" in the key spec pair"); - auto spec_type_str = spec_str.substr(0, spec_delim); - auto spec_data = spec_str.substr(spec_delim + 1); - - auto pubkey = public_key_type(pub_key_str); - - if (spec_type_str == "KEY") { - my->_signature_providers[pubkey] = make_key_signature_provider(private_key_type(spec_data)); - } else if (spec_type_str == "KEOSD") { - my->_signature_providers[pubkey] = make_keosd_signature_provider(my, spec_data, pubkey); - } - + const auto& [pubkey, provider] = app().get_plugin().signature_provider_for_specification(key_spec_pair); + my->_signature_providers[pubkey] = provider; + } catch(secure_enclave_exception& e) { + elog("Error with Secure Enclave signature provider: ${e}; ignoring ${val}", ("e", e.top_message())("val", key_spec_pair)); } catch (...) { elog("Malformed signature provider: \"${val}\", ignoring!", ("val", key_spec_pair)); } } } - my->_keosd_provider_timeout_us = fc::milliseconds(options.at("keosd-provider-timeout").as()); - my->_produce_time_offset_us = options.at("produce-time-offset-us").as(); EOS_ASSERT( my->_produce_time_offset_us <= 0 && my->_produce_time_offset_us >= -config::block_interval_us, plugin_config_exception, "produce-time-offset-us ${o} must be 0 .. -${bi}", ("bi", config::block_interval_us)("o", my->_produce_time_offset_us) ); @@ -917,7 +851,7 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_ EOS_ASSERT( max_incoming_transaction_queue_size > 0, plugin_config_exception, "incoming-transaction-queue-size-mb ${mb} must be greater than 0", ("mb", max_incoming_transaction_queue_size) ); - my->_pending_incoming_transactions.set_max_incoming_transaction_queue_size( max_incoming_transaction_queue_size ); + my->_unapplied_transactions.set_max_transaction_queue_size( max_incoming_transaction_queue_size ); my->_incoming_defer_ratio = options.at("incoming-defer-ratio").as(); @@ -960,6 +894,10 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_ EOS_ASSERT( fc::is_directory(my->_snapshots_dir), snapshot_directory_not_found_exception, "No such directory '${dir}'", ("dir", my->_snapshots_dir.generic_string()) ); + + if (auto resmon_plugin = app().find_plugin()) { + resmon_plugin->monitor_directory(my->_snapshots_dir); + } } my->_incoming_block_subscription = app().get_channel().subscribe( @@ -981,6 +919,11 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_ return my->on_incoming_block(block, block_id); }); + my->_incoming_blockvault_sync_provider = app().get_method().register_provider( + [this](const signed_block_ptr& block, bool check_connectivity) { + return my->on_sync_block(block, check_connectivity); + }); + my->_incoming_transaction_async_provider = app().get_method().register_provider( [this](const packed_transaction_ptr& trx, bool persist_until_expired, next_function next) -> void { return my->on_incoming_transaction_async(trx, persist_until_expired, next ); @@ -1061,8 +1004,15 @@ void producer_plugin::plugin_startup() void producer_plugin::plugin_shutdown() { try { my->_timer.cancel(); - } catch(fc::exception& e) { + my->_block_vault_resync.cancel(); + } catch ( const std::bad_alloc& ) { + chain_plugin::handle_bad_alloc(); + } catch ( const boost::interprocess::bad_alloc& ) { + chain_plugin::handle_bad_alloc(); + } catch(const fc::exception& e) { edump((e.to_detail_string())); + } catch(const std::exception& e) { + edump((fc::std_exception_wrapper::from_current_exception(e).to_detail_string())); } if( my->_thread_pool ) { @@ -1153,7 +1103,7 @@ producer_plugin::runtime_options producer_plugin::get_runtime_options() const { my->_max_scheduled_transaction_time_per_block_ms, my->chain_plug->chain().get_subjective_cpu_leeway() ? my->chain_plug->chain().get_subjective_cpu_leeway()->count() : - fc::optional(), + std::optional(), my->_incoming_defer_ratio, my->chain_plug->chain().get_greylist_limit() }; @@ -1198,12 +1148,12 @@ producer_plugin::whitelist_blacklist producer_plugin::get_whitelist_blacklist() void producer_plugin::set_whitelist_blacklist(const producer_plugin::whitelist_blacklist& params) { chain::controller& chain = my->chain_plug->chain(); - if(params.actor_whitelist.valid()) chain.set_actor_whitelist(*params.actor_whitelist); - if(params.actor_blacklist.valid()) chain.set_actor_blacklist(*params.actor_blacklist); - if(params.contract_whitelist.valid()) chain.set_contract_whitelist(*params.contract_whitelist); - if(params.contract_blacklist.valid()) chain.set_contract_blacklist(*params.contract_blacklist); - if(params.action_blacklist.valid()) chain.set_action_blacklist(*params.action_blacklist); - if(params.key_blacklist.valid()) chain.set_key_blacklist(*params.key_blacklist); + if(params.actor_whitelist) chain.set_actor_whitelist(*params.actor_whitelist); + if(params.actor_blacklist) chain.set_actor_blacklist(*params.actor_blacklist); + if(params.contract_whitelist) chain.set_contract_whitelist(*params.contract_whitelist); + if(params.contract_blacklist) chain.set_contract_blacklist(*params.contract_blacklist); + if(params.action_blacklist) chain.set_action_blacklist(*params.action_blacklist); + if(params.key_blacklist) chain.set_key_blacklist(*params.key_blacklist); } producer_plugin::integrity_hash_information producer_plugin::get_integrity_hash() const { @@ -1227,6 +1177,8 @@ void producer_plugin::create_snapshot(producer_plugin::next_functionchain_plug->chain(); auto head_id = chain.head_block_id(); + const auto head_block_num = chain.head_block_num(); + const auto head_block_time = chain.head_block_time(); const auto& snapshot_path = pending_snapshot::get_final_path(head_id, my->_snapshots_dir); const auto& temp_path = pending_snapshot::get_temp_path(head_id, my->_snapshots_dir); @@ -1269,11 +1221,14 @@ void producer_plugin::create_snapshot(producer_plugin::next_functionblockvault != nullptr ) { + my->blockvault->propose_snapshot( blockvault::watermark_t{head_block_num, head_block_time}, snapshot_path.generic_string().c_str() ); + } } CATCH_AND_CALL (next); return; } @@ -1286,7 +1241,7 @@ void producer_plugin::create_snapshot(producer_plugin::next_function& res){ + entry.next = [prev = entry.next, next](const std::variant& res){ prev(res); next(res); }; @@ -1301,11 +1256,11 @@ void producer_plugin::create_snapshot(producer_plugin::next_function_pending_snapshot_index.emplace(head_id, next, pending_path.generic_string(), snapshot_path.generic_string()); + my->_pending_snapshot_index.emplace(head_id, next, pending_path.generic_string(), snapshot_path.generic_string(), my->blockvault); } CATCH_AND_CALL (next); } } @@ -1415,7 +1370,7 @@ producer_plugin::get_account_ram_corrections( const get_account_ram_corrections_ return result; } -optional producer_plugin_impl::calculate_next_block_time(const account_name& producer_name, const block_timestamp_type& current_block_time) const { +std::optional producer_plugin_impl::calculate_next_block_time(const account_name& producer_name, const block_timestamp_type& current_block_time) const { chain::controller& chain = chain_plug->chain(); const auto& hbs = chain.head_block_state(); const auto& active_schedule = hbs->active_schedule.producers; @@ -1424,7 +1379,7 @@ optional producer_plugin_impl::calculate_next_block_time(const a auto itr = std::find_if(active_schedule.begin(), active_schedule.end(), [&](const auto& asp){ return asp.producer_name == producer_name; }); if (itr == active_schedule.end()) { // this producer is not in the active producer set - return optional(); + return std::optional(); } size_t producer_index = itr - active_schedule.begin(); @@ -1485,8 +1440,12 @@ fc::time_point producer_plugin_impl::calculate_pending_block_time() const { } fc::time_point producer_plugin_impl::calculate_block_deadline( const fc::time_point& block_time ) const { - bool last_block = ((block_timestamp_type(block_time).slot % config::producer_repetitions) == config::producer_repetitions - 1); - return block_time + fc::microseconds(last_block ? _last_block_time_offset_us : _produce_time_offset_us); + if( _pending_block_mode == pending_block_mode::producing ) { + bool last_block = ((block_timestamp_type( block_time ).slot % config::producer_repetitions) == config::producer_repetitions - 1); + return block_time + fc::microseconds(last_block ? _last_block_time_offset_us : _produce_time_offset_us); + } else { + return block_time + fc::microseconds(_produce_time_offset_us); + } } producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { @@ -1497,6 +1456,12 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { const auto& hbs = chain.head_block_state(); + if( chain.get_terminate_at_block() > 0 && chain.get_terminate_at_block() < hbs->block_num ) { + ilog("Reached configured maximum block ${num}; terminating", ("num", chain.get_terminate_at_block())); + app().quit(); + return start_block_result::failed; + } + const fc::time_point now = fc::time_point::now(); const fc::time_point block_time = calculate_pending_block_time(); @@ -1532,6 +1497,9 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { } else if ( _max_irreversible_block_age_us.count() >= 0 && irreversible_block_age >= _max_irreversible_block_age_us ) { elog("Not producing block because the irreversible block is too old [age:${age}s, max:${max}s]", ("age", irreversible_block_age.count() / 1'000'000)( "max", _max_irreversible_block_age_us.count() / 1'000'000 )); _pending_block_mode = pending_block_mode::speculating; + } else if ( _latest_rejected_block_num > hbs->block_num ) { + elog("Not producing block because the block number has been rejected by block vault"); + _pending_block_mode = pending_block_mode::speculating; } if (_pending_block_mode == pending_block_mode::producing) { @@ -1607,10 +1575,18 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { bool drop_features_to_activate = false; try { chain.validate_protocol_features( _protocol_features_to_activate ); + } catch ( const std::bad_alloc& ) { + chain_plugin::handle_bad_alloc(); + } catch ( const boost::interprocess::bad_alloc& ) { + chain_plugin::handle_bad_alloc(); } catch( const fc::exception& e ) { wlog( "protocol features to activate are no longer all valid: ${details}", ("details",e.to_detail_string()) ); drop_features_to_activate = true; + } catch( const std::exception& e ) { + wlog( "protocol features to activate are no longer all valid: ${details}", + ("details",fc::std_exception_wrapper::from_current_exception(e).to_detail_string()) ); + drop_features_to_activate = true; } if( drop_features_to_activate ) { @@ -1650,7 +1626,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { } try { - if( !remove_expired_persisted_trxs( preprocess_deadline ) ) + if( !remove_expired_trxs( preprocess_deadline ) ) return start_block_result::exhausted; if( !remove_expired_blacklisted_trxs( preprocess_deadline ) ) return start_block_result::exhausted; @@ -1658,7 +1634,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { return start_block_result::exhausted; // limit execution of pending incoming to once per block - size_t pending_incoming_process_limit = _pending_incoming_transactions.size(); + size_t pending_incoming_process_limit = _unapplied_transactions.incoming_size(); if( !process_unapplied_trxs( preprocess_deadline ) ) return start_block_result::exhausted; @@ -1699,7 +1675,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { return start_block_result::failed; } -bool producer_plugin_impl::remove_expired_persisted_trxs( const fc::time_point& deadline ) +bool producer_plugin_impl::remove_expired_trxs( const fc::time_point& deadline ) { chain::controller& chain = chain_plug->chain(); auto pending_block_time = chain.pending_block_time(); @@ -1713,24 +1689,16 @@ bool producer_plugin_impl::remove_expired_persisted_trxs( const fc::time_point& &chain, has_producers = !_producers.empty()]( const transaction_id_type& txid, trx_enum_type trx_type ) { if( trx_type == trx_enum_type::persisted ) { if( pbm == pending_block_mode::producing ) { - fc_dlog(_trx_successful_trace_log, - "[TRX_TRACE] Block ${block_num} for producer ${prod} is EXPIRING PERSISTED tx: ${txid}", - ("block_num", chain.head_block_num() + 1)("txid", txid) - ("prod", chain.is_building_block() ? chain.pending_block_producer() : name()) ); fc_dlog(_trx_failed_trace_log, "[TRX_TRACE] Block ${block_num} for producer ${prod} is EXPIRING PERSISTED tx: ${txid}", ("block_num", chain.head_block_num() + 1)("txid", txid) ("prod", chain.is_building_block() ? chain.pending_block_producer() : name()) ); } else { - fc_dlog(_trx_successful_trace_log, "[TRX_TRACE] Speculative execution is EXPIRING PERSISTED tx: ${txid}", ("txid", txid)); fc_dlog(_trx_failed_trace_log, "[TRX_TRACE] Speculative execution is EXPIRING PERSISTED tx: ${txid}", ("txid", txid)); } ++num_expired_persistent; } else { if (has_producers) { - fc_dlog(_trx_successful_trace_log, - "[TRX_TRACE] Node with producers configured is dropping an EXPIRED transaction that was PREVIOUSLY ACCEPTED : ${txid}", - ("txid", txid)); fc_dlog(_trx_failed_trace_log, "[TRX_TRACE] Node with producers configured is dropping an EXPIRED transaction that was PREVIOUSLY ACCEPTED : ${txid}", ("txid", txid)); @@ -1874,10 +1842,11 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin const auto& rl = chain.get_resource_limits_manager(); int num_applied = 0, num_failed = 0, num_processed = 0; auto unapplied_trxs_size = _unapplied_transactions.size(); + // unapplied and persisted do not have a next method to call auto itr = (_pending_block_mode == pending_block_mode::producing) ? - _unapplied_transactions.begin() : _unapplied_transactions.persisted_begin(); + _unapplied_transactions.unapplied_begin() : _unapplied_transactions.persisted_begin(); auto end_itr = (_pending_block_mode == pending_block_mode::producing) ? - _unapplied_transactions.end() : _unapplied_transactions.persisted_end(); + _unapplied_transactions.unapplied_end() : _unapplied_transactions.persisted_end(); while( itr != end_itr ) { if( deadline <= fc::time_point::now() ) { exhausted = true; @@ -1898,7 +1867,7 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin } auto prev_billed_cpu_time_us = trx->billed_cpu_time_us; - if( prev_billed_cpu_time_us > 0 && !rl.is_unlimited_cpu( first_auth )) { + if(!_subjective_billing.is_disabled() && prev_billed_cpu_time_us > 0 && !rl.is_unlimited_cpu( first_auth )) { auto prev_billed_plus100 = prev_billed_cpu_time_us + EOS_PERCENT( prev_billed_cpu_time_us, 100 * config::percent_1 ); auto trx_dl = start + fc::microseconds( prev_billed_plus100 ); if( trx_dl < trx_deadline ) trx_deadline = trx_dl; @@ -1921,6 +1890,7 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin // don't erase, subjective failure so try again next time break; } + // don't erase, subjective failure so try again next time } else { fc_dlog( _trx_failed_trace_log, "Subjective unapplied bill for failed ${a}: ${b} prev ${t}us", ("a",first_auth)("b",prev_billed_cpu_time_us)("t",trace->elapsed)); auto failure_code = trace->except->code(); @@ -1933,6 +1903,7 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin _subjective_billing.subjective_bill_failure( first_auth, trace->elapsed, fc::time_point::now() ); } ++num_failed; + if( itr->next ) itr->next( trace ); itr = _unapplied_transactions.erase( itr ); continue; } @@ -1942,8 +1913,11 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin _subjective_billing.subjective_bill( trx->id(), trx->packed_trx()->expiration(), first_auth, trace->elapsed, chain.get_read_mode() == chain::db_read_mode::SPECULATIVE ); ++num_applied; - itr = _unapplied_transactions.erase( itr ); - continue; + if( itr->trx_type != trx_enum_type::persisted ) { + if( itr->next ) itr->next( trace ); + itr = _unapplied_transactions.erase( itr ); + continue; + } } } LOG_AND_DROP(); ++itr; @@ -1968,6 +1942,8 @@ void producer_plugin_impl::process_scheduled_and_incoming_trxs( const fc::time_p auto& blacklist_by_id = _blacklisted_transactions.get(); chain::controller& chain = chain_plug->chain(); time_point pending_block_time = chain.pending_block_time(); + auto itr = _unapplied_transactions.incoming_begin(); + auto end = _unapplied_transactions.incoming_end(); const auto& sch_idx = chain.db().get_index(); const auto scheduled_trxs_size = sch_idx.size(); auto sch_itr = sch_idx.begin(); @@ -1997,16 +1973,20 @@ void producer_plugin_impl::process_scheduled_and_incoming_trxs( const fc::time_p num_processed++; // configurable ratio of incoming txns vs deferred txns - while (incoming_trx_weight >= 1.0 && pending_incoming_process_limit && _pending_incoming_transactions.size()) { + while (incoming_trx_weight >= 1.0 && pending_incoming_process_limit && itr != end ) { if (deadline <= fc::time_point::now()) { exhausted = true; break; } - auto e = _pending_incoming_transactions.pop_front(); --pending_incoming_process_limit; incoming_trx_weight -= 1.0; - if( !process_incoming_transaction_async(std::get<0>(e), std::get<1>(e), std::get<2>(e)) ) { + + auto trx_meta = itr->trx_meta; + auto next = itr->next; + bool persist_until_expired = itr->trx_type == trx_enum_type::incoming_persisted; + itr = _unapplied_transactions.erase( itr ); + if( !process_incoming_transaction_async( trx_meta, persist_until_expired, next ) ) { exhausted = true; break; } @@ -2032,6 +2012,7 @@ void producer_plugin_impl::process_scheduled_and_incoming_trxs( const fc::time_p exhausted = true; break; } + // do not blacklist } else { // this failed our configured maximum transaction time, we don't want to replay it add it to a blacklist _blacklisted_transactions.insert(transaction_id_with_expiry{trx_id, sch_expiration}); @@ -2059,23 +2040,28 @@ void producer_plugin_impl::process_scheduled_and_incoming_trxs( const fc::time_p bool producer_plugin_impl::process_incoming_trxs( const fc::time_point& deadline, size_t& pending_incoming_process_limit ) { bool exhausted = false; - if (!_pending_incoming_transactions.empty()) { + if( pending_incoming_process_limit ) { size_t processed = 0; - fc_dlog(_log, "Processing ${n} pending transactions", ("n", pending_incoming_process_limit)); - while (pending_incoming_process_limit && _pending_incoming_transactions.size()) { + fc_dlog( _log, "Processing ${n} pending transactions", ("n", pending_incoming_process_limit) ); + auto itr = _unapplied_transactions.incoming_begin(); + auto end = _unapplied_transactions.incoming_end(); + while( pending_incoming_process_limit && itr != end ) { if (deadline <= fc::time_point::now()) { exhausted = true; break; } - auto e = _pending_incoming_transactions.pop_front(); --pending_incoming_process_limit; + auto trx_meta = itr->trx_meta; + auto next = itr->next; + bool persist_until_expired = itr->trx_type == trx_enum_type::incoming_persisted; + itr = _unapplied_transactions.erase( itr ); ++processed; - if( !process_incoming_transaction_async(std::get<0>(e), std::get<1>(e), std::get<2>(e)) ) { + if( !process_incoming_transaction_async( trx_meta, persist_until_expired, next ) ) { exhausted = true; break; } } - fc_dlog(_log, "Processed ${n} pending transactions, ${p} left", ("n", processed)("p", _pending_incoming_transactions.size())); + fc_dlog( _log, "Processed ${n} pending transactions, ${p} left", ("n", processed)("p", _unapplied_transactions.incoming_size()) ); } return !exhausted; } @@ -2173,9 +2159,9 @@ void producer_plugin_impl::schedule_maybe_produce_block( bool exhausted ) { } ) ); } -optional producer_plugin_impl::calculate_producer_wake_up_time( const block_timestamp_type& ref_block_time ) const { +std::optional producer_plugin_impl::calculate_producer_wake_up_time( const block_timestamp_type& ref_block_time ) const { // if we have any producers then we should at least set a timer for our next available slot - optional wake_up_time; + std::optional wake_up_time; for (const auto& p : _producers) { auto next_producer_block_time = calculate_next_block_time(p, ref_block_time); if (next_producer_block_time) { @@ -2197,7 +2183,7 @@ optional producer_plugin_impl::calculate_producer_wake_up_time( return wake_up_time; } -void producer_plugin_impl::schedule_delayed_production_loop(const std::weak_ptr& weak_this, optional wake_up_time) { +void producer_plugin_impl::schedule_delayed_production_loop(const std::weak_ptr& weak_this, std::optional wake_up_time) { if (wake_up_time) { fc_dlog(_log, "Scheduling Speculative/Production Change at ${time}", ("time", wake_up_time)); static const boost::posix_time::ptime epoch(boost::gregorian::date(1970, 1, 1)); @@ -2212,18 +2198,23 @@ void producer_plugin_impl::schedule_delayed_production_loop(const std::weak_ptr< } } - bool producer_plugin_impl::maybe_produce_block() { auto reschedule = fc::make_scoped_exit([this]{ schedule_production_loop(); }); + const char* reason = "produce_block error"; + try { produce_block(); return true; - } LOG_AND_DROP(); + } + catch(block_validation_error&) { + reason = "block vault rejected block, waiting on external block to continue"; + } + LOG_AND_DROP(); - fc_dlog(_log, "Aborting block due to produce_block error"); + fc_wlog(_log, "Aborting block due to ${reason}", ("reason", reason)); abort_block(); return false; } @@ -2235,7 +2226,7 @@ static auto make_debug_time_logger() { }); } -static auto maybe_make_debug_time_logger() -> fc::optional { +static auto maybe_make_debug_time_logger() -> std::optional { if (_log.is_enabled( fc::log_level::debug ) ){ return make_debug_time_logger(); } else { @@ -2243,14 +2234,52 @@ static auto maybe_make_debug_time_logger() -> fc::optionalweak_from_this()](const boost::system::error_code& ec) { + auto shared_impl = weak_impl.lock(); + auto impl = shared_impl.get(); + if (impl && !ec) { + auto id = impl->chain_plug->chain().last_irreversible_block_id(); + fc_dlog(_log, "Attempt to resync from block vault"); + try { + impl->blockvault->sync(&id, *this); + } catch( fc::exception& er ) { + fc_wlog(_log, "Attempting to resync from blockvault encountered ${details}; the node must restart to " + "continue!", + ("details", er.to_detail_string())); + app().quit(); + } + } + this->_pending = false; + })); + } +} + +void block_only_sync::on_snapshot(const char*) { + EOS_THROW(producer_exception, "a snapshot"); +} + +void block_only_sync::on_block(eosio::chain::signed_block_ptr block) { + try { + bool connectivity_check = false; // use false right now, should investigate further after 3.0 rc + _impl->on_sync_block(block, connectivity_check); + } + catch (const unlinkable_block_exception&) { + fc_dlog(_log, "got unlinkable block ${num} from block vault", ("num", block->block_num())); + } +} + void producer_plugin_impl::produce_block() { //ilog("produce_block ${t}", ("t", fc::time_point::now())); // for testing _produce_time_offset_us EOS_ASSERT(_pending_block_mode == pending_block_mode::producing, producer_exception, "called produce_block while not actually producing"); chain::controller& chain = chain_plug->chain(); - const auto& hbs = chain.head_block_state(); EOS_ASSERT(chain.is_building_block(), missing_pending_block_state, "pending_block_state does not exist but it should, another plugin may have corrupted it"); - const auto& auth = chain.pending_block_signing_authority(); std::vector> relevant_providers; @@ -2271,7 +2300,7 @@ void producer_plugin_impl::produce_block() { } //idump( (fc::time_point::now() - chain.pending_block_time()) ); - chain.finalize_block( [&]( const digest_type& d ) { + block_state_ptr pending_blk_state = chain.finalize_block( [&]( const digest_type& d ) { auto debug_logger = maybe_make_debug_time_logger(); vector sigs; sigs.reserve(relevant_providers.size()); @@ -2283,15 +2312,25 @@ void producer_plugin_impl::produce_block() { return sigs; } ); - chain.commit_block(); + if ( blockvault != nullptr ) { + std::promise p; + std::future f = p.get_future(); + blockvault->async_propose_constructed_block(pending_blk_state->dpos_irreversible_blocknum, + pending_blk_state->block, [&p](bool b) { p.set_value(b); }); + if (!f.get()) { + _latest_rejected_block_num = pending_blk_state->block->block_num(); + _block_vault_resync.schedule(); + EOS_ASSERT(false, block_validation_error, "Block rejected by block vault"); + } - block_state_ptr new_bs = chain.head_block_state(); + } + chain.commit_block(); + block_state_ptr new_bs = chain.head_block_state(); ilog("Produced block ${id}... #${n} @ ${t} signed by ${p} [trxs: ${count}, lib: ${lib}, confirmed: ${confs}]", ("p",new_bs->header.producer)("id",new_bs->id.str().substr(8,16)) ("n",new_bs->block_num)("t",new_bs->header.timestamp) ("count",new_bs->block->transactions.size())("lib",chain.last_irreversible_block_num())("confs", new_bs->header.confirmed)); - } void producer_plugin::log_failed_transaction(const transaction_id_type& trx_id, const char* reason) const { diff --git a/plugins/producer_plugin/test/CMakeLists.txt b/plugins/producer_plugin/test/CMakeLists.txt index 689dca3981f..2278a08b40f 100644 --- a/plugins/producer_plugin/test/CMakeLists.txt +++ b/plugins/producer_plugin/test/CMakeLists.txt @@ -1,3 +1,8 @@ +add_executable( test_snapshot_information test_snapshot_information.cpp ) +target_link_libraries( test_snapshot_information producer_plugin eosio_testing ) + +add_test(NAME test_snapshot_information COMMAND plugins/producer_plugin/test/test_snapshot_information WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + add_executable( test_subjective_billing test_subjective_billing.cpp ) target_link_libraries( test_subjective_billing producer_plugin eosio_testing ) diff --git a/plugins/producer_plugin/test/test_snapshot_information.cpp b/plugins/producer_plugin/test/test_snapshot_information.cpp new file mode 100644 index 00000000000..67861d40f12 --- /dev/null +++ b/plugins/producer_plugin/test/test_snapshot_information.cpp @@ -0,0 +1,71 @@ +#define BOOST_TEST_MODULE snapshot_information +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace eosio; +using namespace eosio::testing; +using namespace boost::system; + +namespace { + eosio::producer_plugin::snapshot_information test_snap_info; +} + +BOOST_AUTO_TEST_SUITE(snapshot_tests) + +using next_t = eosio::producer_plugin::next_function; + +BOOST_AUTO_TEST_CASE_TEMPLATE(test_snapshot_information, SNAPSHOT_SUITE, snapshot_suites) { + tester chain; + const chainbase::bfs::path parent_path = chain.get_config().blog.log_dir.parent_path(); + + chain.create_account("snapshot"_n); + chain.produce_blocks(1); + chain.set_code("snapshot"_n, contracts::snapshot_test_wasm()); + chain.set_abi("snapshot"_n, contracts::snapshot_test_abi().data()); + chain.produce_blocks(1); + + auto block = chain.produce_block(); + BOOST_REQUIRE_EQUAL(block->block_num(), 6); // ensure that test setup stays consistent with original snapshot setup + // undo the auto-pending from tester + chain.control->abort_block(); + + auto block2 = chain.produce_block(); + BOOST_REQUIRE_EQUAL(block2->block_num(), 7); // ensure that test setup stays consistent with original snapshot setup + // undo the auto-pending from tester + chain.control->abort_block(); + + // write snapshot + auto write_snapshot = [&]( const bfs::path& p ) -> void { + if ( !bfs::exists( p.parent_path() ) ) + bfs::create_directory( p.parent_path() ); + + // create the snapshot + auto snap_out = std::ofstream(p.generic_string(), (std::ios::out | std::ios::binary)); + auto writer = std::make_shared(snap_out); + (*chain.control).write_snapshot(writer); + writer->finalize(); + snap_out.flush(); + snap_out.close(); + }; + + auto final_path = eosio::pending_snapshot::get_final_path(block2->previous, "../snapshots/"); + auto pending_path = eosio::pending_snapshot::get_pending_path(block2->previous, "../snapshots/"); + + write_snapshot( pending_path ); + next_t next; + eosio::pending_snapshot pending{ block2->previous, next, pending_path.generic_string(), final_path.generic_string(), nullptr }; + test_snap_info = pending.finalize(*chain.control); + BOOST_REQUIRE_EQUAL(test_snap_info.head_block_num, 6); + BOOST_REQUIRE_EQUAL(test_snap_info.version, chain_snapshot_header::current_version); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/plugins/producer_plugin/test/test_subjective_billing.cpp b/plugins/producer_plugin/test/test_subjective_billing.cpp index dfccb54973d..40ea4d21c5d 100644 --- a/plugins/producer_plugin/test/test_subjective_billing.cpp +++ b/plugins/producer_plugin/test/test_subjective_billing.cpp @@ -22,9 +22,9 @@ BOOST_AUTO_TEST_CASE( subjective_bill_test ) { transaction_id_type id4 = sha256::hash( "4" ); transaction_id_type id5 = sha256::hash( "5" ); transaction_id_type id6 = sha256::hash( "6" ); - account_name a = N("a"); - account_name b = N("b"); - account_name c = N("c"); + account_name a = "a"_n; + account_name b = "b"_n; + account_name c = "c"_n; const auto now = time_point::now(); const auto halftime = now + fc::milliseconds(subjective_billing::expired_accumulator_average_window * subjective_billing::subjective_time_interval_ms / 2); diff --git a/plugins/resource_monitor_plugin/CMakeLists.txt b/plugins/resource_monitor_plugin/CMakeLists.txt new file mode 100644 index 00000000000..fff4bb30ae7 --- /dev/null +++ b/plugins/resource_monitor_plugin/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB HEADERS "include/eosio/resource_monitor_plugin/*.hpp") +add_library( resource_monitor_plugin + resource_monitor_plugin.cpp + system_file_space_provider.cpp + ${HEADERS} ) + +target_link_libraries( resource_monitor_plugin appbase fc chain_plugin) +target_include_directories( resource_monitor_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) + +add_subdirectory( test ) diff --git a/plugins/resource_monitor_plugin/include/eosio/resource_monitor_plugin/file_space_handler.hpp b/plugins/resource_monitor_plugin/include/eosio/resource_monitor_plugin/file_space_handler.hpp new file mode 100644 index 00000000000..994dd5780d0 --- /dev/null +++ b/plugins/resource_monitor_plugin/include/eosio/resource_monitor_plugin/file_space_handler.hpp @@ -0,0 +1,181 @@ +#pragma once + +#include +#include + +#include +#include + +namespace bfs = boost::filesystem; + +namespace eosio::resource_monitor { + template + class file_space_handler { + public: + file_space_handler(SpaceProvider&& space_provider, boost::asio::io_context& ctx) + :space_provider(std::move(space_provider)), + timer{ctx} + { + } + + void set_sleep_time(uint32_t sleep_time) { + sleep_time_in_secs = sleep_time; + } + + // warning_threshold must be less than shutdown_threshold. + // set them together so it is simpler to check. + void set_threshold(uint32_t new_threshold, uint32_t new_warning_threshold) { + EOS_ASSERT(new_warning_threshold < new_threshold, chain::plugin_config_exception, + "warning_threshold ${new_warning_threshold} must be less than threshold ${new_threshold}", ("new_warning_threshold", new_warning_threshold) ("new_threshold", new_threshold)); + + shutdown_threshold = new_threshold; + warning_threshold = new_warning_threshold; + } + + void set_shutdown_on_exceeded(bool new_shutdown_on_exceeded) { + shutdown_on_exceeded = new_shutdown_on_exceeded; + } + + void set_warning_interval(uint32_t new_warning_interval) { + warning_interval = new_warning_interval; + } + + bool is_threshold_exceeded() { + // Go over each monitored file system + for (auto& fs: filesystems) { + boost::system::error_code ec; + auto info = space_provider.get_space(fs.path_name, ec); + if ( ec ) { + // As the system is running and this plugin is not a critical + // part of the system, we should not exit. + // Just report the failure and continue; + wlog( "Unable to get space info for ${path_name}: [code: ${ec}] ${message}. Ignore this failure.", + ("path_name", fs.path_name.string()) + ("ec", ec.value()) + ("message", ec.message())); + + continue; + } + + if ( info.available < fs.shutdown_available ) { + if (output_threshold_warning) { + wlog("Space usage warning: ${path}'s file system exceeded threshold ${threshold}%, available: ${available}, Capacity: ${capacity}, shutdown_available: ${shutdown_available}", ("path", fs.path_name.string()) ("threshold", shutdown_threshold) ("available", info.available) ("capacity", info.capacity) ("shutdown_available", fs.shutdown_available)); + } + return true; + } else if ( info.available < fs.warning_available && output_threshold_warning ) { + wlog("Space usage warning: ${path}'s file system approaching threshold. available: ${available}, warning_available: ${warning_available}", ("path", fs.path_name.string()) ("available", info.available) ("warning_available", fs.warning_available)); + if ( shutdown_on_exceeded) { + wlog("nodeos will shutdown when space usage exceeds threshold ${threshold}%", ("threshold", shutdown_threshold)); + } + } + } + + return false; + } + + void add_file_system(const bfs::path& path_name) { + // Get detailed information of the path + struct stat statbuf; + auto status = space_provider.get_stat(path_name.string().c_str(), &statbuf); + EOS_ASSERT(status == 0, chain::plugin_config_exception, + "Failed to run stat on ${path} with status ${status}", ("path", path_name.string())("status", status)); + + dlog("${path_name}'s file system to be monitored", ("path_name", path_name.string())); + + // If the file system containing the path is already + // in the filesystem list, do not add it again + for (auto& fs: filesystems) { + if (statbuf.st_dev == fs.st_dev) { // Two files belong to the same file system if their device IDs are the same. + dlog("${path_name}'s file system already monitored", ("path_name", path_name.string())); + + return; + } + } + + // For efficiency, precalculate threshold values to avoid calculating it + // everytime we check space usage. Since bfs::space returns + // available amount, we use minimum available amount as threshold. + boost::system::error_code ec; + auto info = space_provider.get_space(path_name, ec); + EOS_ASSERT(!ec, chain::plugin_config_exception, + "Unable to get space info for ${path_name}: [code: ${ec}] ${message}", + ("path_name", path_name.string()) + ("ec", ec.value()) + ("message", ec.message())); + + auto shutdown_available = (100 - shutdown_threshold) * (info.capacity / 100); // (100 - shutdown_threshold)/100 is the percentage of minimum number of available bytes the file system must maintain + auto warning_available = (100 - warning_threshold) * (info.capacity / 100); + + // Add to the list + filesystems.emplace_back(statbuf.st_dev, shutdown_available, path_name, warning_available); + + ilog("${path_name}'s file system monitored. shutdown_available: ${shutdown_available}, capacity: ${capacity}, threshold: ${threshold}", ("path_name", path_name.string()) ("shutdown_available", shutdown_available) ("capacity", info.capacity) ("threshold", shutdown_threshold) ); + } + + void space_monitor_loop() { + if ( is_threshold_exceeded() && shutdown_on_exceeded ) { + wlog("Shutting down"); + appbase::app().quit(); // This will gracefully stop Nodeos + return; + } + update_warning_interval_counter(); + + timer.expires_from_now( boost::posix_time::seconds( sleep_time_in_secs )); + + timer.async_wait([this](auto& ec) { + if ( ec ) { + wlog("Exit due to error: ${ec}, message: ${message}", + ("ec", ec.value()) + ("message", ec.message())); + return; + } else { + // Loop over + space_monitor_loop(); + } + }); + } + + private: + SpaceProvider space_provider; + + boost::asio::deadline_timer timer; + + uint32_t sleep_time_in_secs {2}; + uint32_t shutdown_threshold {90}; + uint32_t warning_threshold {85}; + bool shutdown_on_exceeded {true}; + + struct filesystem_info { + dev_t st_dev; // device id of file system containing "file_path" + uintmax_t shutdown_available {0}; // minimum number of available bytes the file system must maintain + bfs::path path_name; + uintmax_t warning_available {0}; // warning is issued when availabla number of bytese drops below warning_available + + filesystem_info(dev_t dev, uintmax_t available, const bfs::path& path, uintmax_t warning) + : st_dev(dev), + shutdown_available(available), + path_name(path), + warning_available(warning) + { + } + }; + + // Stores file systems to be monitored. Duplicate + // file systems are not stored. + std::vector filesystems; + + uint32_t warning_interval {1}; + uint32_t warning_interval_counter {1}; + bool output_threshold_warning {true}; + + void update_warning_interval_counter() { + if ( warning_interval_counter == warning_interval ) { + output_threshold_warning = true; + warning_interval_counter = 1; + } else { + output_threshold_warning = false; + ++warning_interval_counter; + } + } + }; +} diff --git a/plugins/resource_monitor_plugin/include/eosio/resource_monitor_plugin/resource_monitor_plugin.hpp b/plugins/resource_monitor_plugin/include/eosio/resource_monitor_plugin/resource_monitor_plugin.hpp new file mode 100644 index 00000000000..739ef99b0d4 --- /dev/null +++ b/plugins/resource_monitor_plugin/include/eosio/resource_monitor_plugin/resource_monitor_plugin.hpp @@ -0,0 +1,29 @@ +#pragma once +#include +#include + +namespace eosio { + +using namespace appbase; + +class resource_monitor_plugin : public appbase::plugin { +public: + resource_monitor_plugin( ); + virtual ~resource_monitor_plugin(); + + APPBASE_PLUGIN_REQUIRES( (chain_plugin) ) + virtual void set_program_options(options_description&, options_description& cfg) override; + + void plugin_initialize(const variables_map& options); + void plugin_startup(); + void plugin_shutdown(); + + // Called by plugins and other components to request + // directory monitoring + void monitor_directory(const bfs::path& path); + +private: + std::unique_ptr my; +}; + +} diff --git a/plugins/resource_monitor_plugin/include/eosio/resource_monitor_plugin/system_file_space_provider.hpp b/plugins/resource_monitor_plugin/include/eosio/resource_monitor_plugin/system_file_space_provider.hpp new file mode 100644 index 00000000000..f76f974fc86 --- /dev/null +++ b/plugins/resource_monitor_plugin/include/eosio/resource_monitor_plugin/system_file_space_provider.hpp @@ -0,0 +1,21 @@ +#pragma once + +#include +#include + +namespace bfs = boost::filesystem; + +namespace eosio::resource_monitor { + class system_file_space_provider { + public: + system_file_space_provider() + { + } + + // Wrapper for Linux stat + int get_stat(const char *path, struct stat *buf) const; + + // Wrapper for boost file system space + bfs::space_info get_space(const bfs::path& p, boost::system::error_code& ec) const; + }; +} diff --git a/plugins/resource_monitor_plugin/resource_monitor_plugin.cpp b/plugins/resource_monitor_plugin/resource_monitor_plugin.cpp new file mode 100644 index 00000000000..3e8894fe782 --- /dev/null +++ b/plugins/resource_monitor_plugin/resource_monitor_plugin.cpp @@ -0,0 +1,184 @@ +/** + It was reported from a customer that when file system which + "data/blocks" belongs to is running out of space, the producer + continued to produce blocks and update state but the blocks log was + "corrupted" in that it no longer contained all the irreversible blocks. + It was also observed that when file system which "data/state" + belons to is running out of space, nodeos will crash with SIGBUS as + the state file is unable to acquire new pages. + + The solution is to have a dedicated plugin to monitor resource + usages (file system space now, CPU, memory, and networking + bandwidth in the future). + The plugin uses a thread to periodically check space usage of file + systems of directories being monitored. If space used + is over a predefined threshold, a graceful shutdown is initiated. +**/ + +#include +#include +#include + +#include + +#include +#include // set_os_thread_name + +#include + +#include + +#include + +using namespace eosio::resource_monitor; + +namespace bfs = boost::filesystem; + +namespace eosio { + static appbase::abstract_plugin& _resource_monitor_plugin = app().register_plugin(); + +class resource_monitor_plugin_impl { +public: + resource_monitor_plugin_impl() + :space_handler(system_file_space_provider(), ctx) + { + } + + void set_program_options(options_description&, options_description& cfg) { + cfg.add_options() + ( "resource-monitor-interval-seconds", bpo::value()->default_value(def_interval_in_secs), + "Time in seconds between two consecutive checks of resource usage. Should be between 1 and 300" ) + ( "resource-monitor-space-threshold", bpo::value()->default_value(def_space_threshold), + "Threshold in terms of percentage of used space vs total space. If used space is above (threshold - 5%), a warning is generated. If used space is above the threshold and resource-monitor-not-shutdown-on-threshold-exceeded is enabled, a graceful shutdown is initiated. The value should be between 6 and 99" ) + ( "resource-monitor-not-shutdown-on-threshold-exceeded", + "Used to indicate nodeos will not shutdown when threshold is exceeded." ) + ( "resource-monitor-warning-interval", bpo::value()->default_value(def_monitor_warning_interval), + "Number of resource monitor intervals between two consecutive warnings when the threshold is hit. Should be between 1 and 450" ) + ; + } + + void plugin_initialize(const appbase::variables_map& options) { + dlog("plugin_initialize"); + + auto interval = options.at("resource-monitor-interval-seconds").as(); + EOS_ASSERT(interval >= monitor_interval_min && interval <= monitor_interval_max, chain::plugin_config_exception, + "\"resource-monitor-interval-seconds\" must be between ${monitor_interval_min} and ${monitor_interval_max}", ("monitor_interval_min", monitor_interval_min) ("monitor_interval_max", monitor_interval_max)); + space_handler.set_sleep_time(interval); + ilog("Monitoring interval set to ${interval}", ("interval", interval)); + + auto threshold = options.at("resource-monitor-space-threshold").as(); + EOS_ASSERT(threshold >= space_threshold_min && threshold <= space_threshold_max, chain::plugin_config_exception, + "\"resource-monitor-space-threshold\" must be between ${space_threshold_min} and ${space_threshold_max}", ("space_threshold_min", space_threshold_min) ("space_threshold_max", space_threshold_max)); + space_handler.set_threshold(threshold, threshold - space_threshold_warning_diff); + ilog("Space usage threshold set to ${threshold}", ("threshold", threshold)); + + if (options.count("resource-monitor-not-shutdown-on-threshold-exceeded")) { + // If set, not shutdown + space_handler.set_shutdown_on_exceeded(false); + ilog("Shutdown flag when threshold exceeded set to false"); + } else { + // Default will shut down + space_handler.set_shutdown_on_exceeded(true); + ilog("Shutdown flag when threshold exceeded set to true"); + } + + auto warning_interval = options.at("resource-monitor-warning-interval").as(); + EOS_ASSERT(warning_interval >= warning_interval_min && warning_interval <= warning_interval_max, chain::plugin_config_exception, + "\"resource-monitor-warning-interval\" must be between ${warning_interval_min} and ${warning_interval_max}", ("warning_interval_min", warning_interval_min) ("warning_interval_max", warning_interval_max)); + space_handler.set_warning_interval(warning_interval); + ilog("Warning interval set to ${warning_interval}", ("warning_interval", warning_interval)); + } + + // Start main thread + void plugin_startup() { + ilog("Creating and starting monitor thread"); + + // By now all plugins are initialized. + // Find out filesystems containing the directories requested + // so far. + for ( auto& dir: directories_registered ) { + space_handler.add_file_system( dir ); + + // A directory like "data" contains subdirectories like + // "block". Those subdirectories can mount on different + // file systems. Make sure they are taken care of. + for (bfs::directory_iterator itr(dir); itr != bfs::directory_iterator(); ++itr) { + if (fc::is_directory(itr->path())) { + space_handler.add_file_system( itr->path() ); + } + } + } + + monitor_thread = std::thread( [this] { + fc::set_os_thread_name( "resmon" ); // console_appender uses 9 chars for thread name reporting. + space_handler.space_monitor_loop(); + + ctx.run(); + } ); + } + + // System is shutting down. + void plugin_shutdown() { + ilog("shutdown..."); + + ctx.stop(); + + // Wait for the thread to end + monitor_thread.join(); + + ilog("exit shutdown"); + } + + void monitor_directory(const bfs::path& path) { + dlog("${path} registered to be monitored", ("path", path.string())); + directories_registered.push_back(path); + } + +private: + std::thread monitor_thread; + std::vector directories_registered; + + static constexpr uint32_t def_interval_in_secs = 2; + static constexpr uint32_t monitor_interval_min = 1; + static constexpr uint32_t monitor_interval_max = 300; + + static constexpr uint32_t def_space_threshold = 90; // in percentage + static constexpr uint32_t space_threshold_min = 6; // in percentage + static constexpr uint32_t space_threshold_max = 99; // in percentage + static constexpr uint32_t space_threshold_warning_diff = 5; // Warning issued when space used reached (threshold - space_threshold_warning_diff). space_threshold_warning_diff must be smaller than space_threshold_min + + static constexpr uint32_t def_monitor_warning_interval = 30; // After this number of monitor intervals, warning is output if the threshold is hit + static constexpr uint32_t warning_interval_min = 1; + static constexpr uint32_t warning_interval_max = 450; // e.g. if the monitor interval is 2 sec, the warning interval is at most 15 minutes + + boost::asio::io_context ctx; + + using file_space_handler_t = file_space_handler; + file_space_handler_t space_handler; +}; + +resource_monitor_plugin::resource_monitor_plugin():my(std::make_unique()) {} + +resource_monitor_plugin::~resource_monitor_plugin() {} + +void resource_monitor_plugin::set_program_options(options_description& cli, options_description& cfg) { + my->set_program_options(cli, cfg); +} + +void resource_monitor_plugin::plugin_initialize(const variables_map& options) { + my->plugin_initialize(options); +} + +void resource_monitor_plugin::plugin_startup() { + my->plugin_startup(); +} + +void resource_monitor_plugin::plugin_shutdown() { + my->plugin_shutdown(); +} + +void resource_monitor_plugin::monitor_directory(const bfs::path& path) { + my->monitor_directory( path ); +} + +} // namespace diff --git a/plugins/resource_monitor_plugin/system_file_space_provider.cpp b/plugins/resource_monitor_plugin/system_file_space_provider.cpp new file mode 100644 index 00000000000..6086bd871c2 --- /dev/null +++ b/plugins/resource_monitor_plugin/system_file_space_provider.cpp @@ -0,0 +1,15 @@ +#include + +namespace bfs = boost::filesystem; + +namespace eosio::resource_monitor { + int system_file_space_provider::get_stat(const char *path, struct stat *buf) const { + return stat(path, buf); + } + + bfs::space_info system_file_space_provider::get_space(const bfs::path& p, boost::system::error_code& ec) const { + return bfs::space(p, ec); + } + + using bfs::directory_iterator; +} diff --git a/plugins/resource_monitor_plugin/test/CMakeLists.txt b/plugins/resource_monitor_plugin/test/CMakeLists.txt new file mode 100644 index 00000000000..b22bcdc647e --- /dev/null +++ b/plugins/resource_monitor_plugin/test/CMakeLists.txt @@ -0,0 +1,22 @@ +add_executable( test_threshold test_threshold.cpp ) +target_link_libraries( test_threshold resource_monitor_plugin ) +target_include_directories( test_threshold PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) + +add_test(NAME test_threshold COMMAND plugins/resource_monitor_plugin/test/test_threshold WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + +add_executable( test_monitor_loop test_monitor_loop.cpp ) +target_link_libraries( test_monitor_loop resource_monitor_plugin ) +target_include_directories( test_monitor_loop PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) + +add_test(NAME test_monitor_loop COMMAND plugins/resource_monitor_plugin/test/test_monitor_loop WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + +add_executable( test_add_file_system test_add_file_system.cpp ) +target_link_libraries( test_add_file_system resource_monitor_plugin ) +target_include_directories( test_add_file_system PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) + +add_test(NAME test_add_file_system COMMAND plugins/resource_monitor_plugin/test/test_add_file_system WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + +add_executable( test_resmon_plugin test_resmon_plugin.cpp ) +target_link_libraries( test_resmon_plugin resource_monitor_plugin ) +target_include_directories( test_resmon_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) +add_test(NAME test_resmon_plugin COMMAND plugins/resource_monitor_plugin/test/test_resmon_plugin WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/plugins/resource_monitor_plugin/test/test_add_file_system.cpp b/plugins/resource_monitor_plugin/test/test_add_file_system.cpp new file mode 100644 index 00000000000..f04ce9a8c86 --- /dev/null +++ b/plugins/resource_monitor_plugin/test/test_add_file_system.cpp @@ -0,0 +1,138 @@ +#define BOOST_TEST_MODULE add_file_system +#include + +#include + +#include + +using namespace eosio; +using namespace eosio::resource_monitor; +using namespace boost::system; + +struct add_file_system_fixture { + struct mock_space_provider { + mock_space_provider(add_file_system_fixture& fixture) + :fixture(fixture) + {} + + int get_stat(const char *path, struct stat *buf) const { + return fixture.mock_get_stat(path, buf); + } + + bfs::space_info get_space(const bfs::path& p, boost::system::error_code& ec) const { + return fixture.mock_get_space(p, ec); + } + + add_file_system_fixture& fixture; + }; + + boost::asio::io_context ctx; + + using file_space_handler_t = file_space_handler; + add_file_system_fixture() + : space_handler(mock_space_provider(*this), ctx) + { + } + + void add_file_system(const bfs::path& path_name) { + space_handler.add_file_system(path_name); + } + + void set_threshold(uint32_t threshold, uint32_t warning_threshold) { + space_handler.set_threshold( threshold, warning_threshold ); + } + + bool is_threshold_exceeded() { + return space_handler.is_threshold_exceeded(); + } + + void test_add_file_systems_common(std::vector& capacity, std::vector& available, std::vector& devs) { + mock_get_space = [ i = 0, capacity, available ]( const bfs::path& p, boost::system::error_code& ec) mutable -> bfs::space_info { + ec = boost::system::errc::make_error_code(errc::success); + + bfs::space_info rc; + rc.capacity = capacity[i]; + rc.available = available[i]; + i++; + + return rc; + }; + + mock_get_stat = [ j = 0, devs ]( const char *path, struct stat *buf ) mutable -> int { + buf->st_dev = devs[j]; + j++; + + return 0; + }; + + set_threshold(80, 75); + + for (auto k = 0U; k < capacity.size(); k++) { + add_file_system("/test" + std::to_string(k)); + } + } + + // fixture data and methods + std::function mock_get_space; + std::function mock_get_stat; + + file_space_handler_t space_handler; +}; + +BOOST_AUTO_TEST_SUITE(space_handler_tests) + BOOST_FIXTURE_TEST_CASE(get_stat_failure, add_file_system_fixture) + { + mock_get_stat = []( const char *path, struct stat *buf ) -> int { + return 1; // anything other than 0 is an error in stat + }; + + BOOST_REQUIRE_THROW(add_file_system("/test"), chain::plugin_config_exception); + } + + BOOST_FIXTURE_TEST_CASE(get_space_failure, add_file_system_fixture) + { + mock_get_space = []( const bfs::path& p, boost::system::error_code& ec) -> bfs::space_info { + ec = boost::system::errc::make_error_code(errc::no_such_file_or_directory); + bfs::space_info rc; + return rc; + }; + + mock_get_stat = []( const char *path, struct stat *buf ) -> int { + buf->st_dev = 0; + return 0; + }; + + BOOST_REQUIRE_THROW(add_file_system("/test"), chain::plugin_config_exception); + } + + BOOST_FIXTURE_TEST_CASE(different_file_systems, add_file_system_fixture) + { + std::vector capacity {1000000, 2000000, 3000000, 4000000}; + std::vector available {500000, 1500000, 2500000, 3500000}; + std::vector devs {0, 1, 2, 3}; + + // As long as no exceptions, it is considered success. + BOOST_REQUIRE_NO_THROW(test_add_file_systems_common(capacity, available, devs)); + } + + BOOST_FIXTURE_TEST_CASE(same_file_system, add_file_system_fixture) + { + std::vector capacity {1000000, 2000000, 3000000, 4000000}; + std::vector available {500000, 1500000, 2500000, 3500000}; + std::vector devs {0, 0, 0, 0}; + + // As long as no exceptions, it is considered success. + BOOST_REQUIRE_NO_THROW(test_add_file_systems_common(capacity, available, devs)); + } + + BOOST_FIXTURE_TEST_CASE(mixed_file_systems, add_file_system_fixture) + { + std::vector capacity {1000000, 2000000, 3000000, 4000000, 50000}; + std::vector available {500000, 1500000, 2500000, 3500000, 20000}; + std::vector devs {0, 2, 2, 0, 3}; + + // As long as no exceptions, it is considered success. + BOOST_REQUIRE_NO_THROW(test_add_file_systems_common(capacity, available, devs)); + } + +BOOST_AUTO_TEST_SUITE_END() diff --git a/plugins/resource_monitor_plugin/test/test_monitor_loop.cpp b/plugins/resource_monitor_plugin/test/test_monitor_loop.cpp new file mode 100644 index 00000000000..69400056f2e --- /dev/null +++ b/plugins/resource_monitor_plugin/test/test_monitor_loop.cpp @@ -0,0 +1,159 @@ +#define BOOST_TEST_MODULE monitor_loop +#include + +#include + +#include + +using namespace eosio; +using namespace eosio::resource_monitor; +using namespace boost::system; + +struct space_handler_fixture { + struct mock_space_provider { + mock_space_provider(space_handler_fixture& fixture) + :fixture(fixture) + {} + + int get_stat(const char *path, struct stat *buf) const { + return fixture.mock_get_stat(path, buf); + } + + bfs::space_info get_space(const bfs::path& p, boost::system::error_code& ec) const { + return fixture.mock_get_space(p, ec); + } + + space_handler_fixture& fixture; + }; + + boost::asio::io_context ctx; + + using file_space_handler_t = file_space_handler; + space_handler_fixture() + : space_handler(mock_space_provider( *this ), ctx) + { + } + + void add_file_system(const bfs::path& path_name) { + space_handler.add_file_system( path_name ); + } + + void set_threshold(uint32_t threshold, uint32_t warning_threshold) { + space_handler.set_threshold( threshold, warning_threshold ); + } + + void set_sleep_time(uint32_t sleep_time) { + space_handler.set_sleep_time( sleep_time ); + } + + void set_shutdown_on_exceeded(bool shutdown_on_exceeded) { + space_handler.set_shutdown_on_exceeded(shutdown_on_exceeded); + } + + bool is_threshold_exceeded() { + return space_handler.is_threshold_exceeded(); + } + + void space_monitor_loop() { + return space_handler.space_monitor_loop(); + } + + bool test_loop_common(int num_loops, int interval) + { + mock_get_space = [ i = 0, num_loops ]( const bfs::path& p, boost::system::error_code& ec) mutable -> bfs::space_info { + ec = boost::system::errc::make_error_code(errc::success); + + bfs::space_info rc; + rc.capacity = 1000000; + + if ( i < num_loops + 1 ) { // "+ 1" for the get_space in add_file_system + rc.available = 300000; + } else { + rc.available = 100000; + } + + i++; + + return rc; + }; + + mock_get_stat = []( const char *path, struct stat *buf ) -> int { + buf->st_dev = 0; + return 0; + }; + + set_threshold(80, 75); + set_shutdown_on_exceeded(true); + set_sleep_time(interval); + add_file_system("/test"); + + auto start = std::chrono::system_clock::now(); + + auto monitor_thread = std::thread( [this] { + space_monitor_loop(); + ctx.run(); + }); + + monitor_thread.join(); + + auto end = std::chrono::system_clock::now(); + std::chrono::duration test_duration = end - start; + + // For tests to be repeatable on any platforms under any loads, + // particularly for longer runs, + // we just make sure the test duration is longer than a margin + // of theroretical duration. + bool finished_in_time = (test_duration >= std::chrono::duration((num_loops - 1) * interval)); + + return finished_in_time; + } + + // fixture data and methods + std::function mock_get_space; + std::function mock_get_stat; + + file_space_handler_t space_handler; +}; + +BOOST_AUTO_TEST_SUITE(monitor_loop_tests) + BOOST_FIXTURE_TEST_CASE(zero_loop, space_handler_fixture) + { + BOOST_TEST( test_loop_common(0, 1) ); + } + + BOOST_FIXTURE_TEST_CASE(one_loop_1_secs_interval, space_handler_fixture) + { + BOOST_TEST( test_loop_common(1, 1) ); + } + + BOOST_FIXTURE_TEST_CASE(two_loops_1_sec_interval, space_handler_fixture) + { + BOOST_TEST( test_loop_common(2, 1) ); + } + + BOOST_FIXTURE_TEST_CASE(ten_loops_1_sec_interval, space_handler_fixture) + { + BOOST_TEST( test_loop_common(10, 1) ); + } + + BOOST_FIXTURE_TEST_CASE(one_loop_5_secs_interval, space_handler_fixture) + { + BOOST_TEST( test_loop_common(1, 5) ); + } + + BOOST_FIXTURE_TEST_CASE(two_loops_5_sec_interval, space_handler_fixture) + { + BOOST_TEST( test_loop_common(2, 5) ); + } + + BOOST_FIXTURE_TEST_CASE(ten_loops_5_sec_interval, space_handler_fixture) + { + BOOST_TEST( test_loop_common(10, 5) ); + } + + BOOST_FIXTURE_TEST_CASE(one_hundred_twenty_loops_1_sec_interval, space_handler_fixture) + { + BOOST_TEST( test_loop_common(120, 1) ); + } + +BOOST_AUTO_TEST_SUITE_END() diff --git a/plugins/resource_monitor_plugin/test/test_resmon_plugin.cpp b/plugins/resource_monitor_plugin/test/test_resmon_plugin.cpp new file mode 100644 index 00000000000..cea24e66b33 --- /dev/null +++ b/plugins/resource_monitor_plugin/test/test_resmon_plugin.cpp @@ -0,0 +1,173 @@ +#define BOOST_TEST_MODULE test_resmom_plugin +#include + +#include + +#include + +#include + +using namespace eosio; +using namespace boost::system; + +namespace bfs = boost::filesystem; + +// For program options +namespace bpo = boost::program_options; +using bpo::options_description; +using bpo::variables_map; + +struct resmon_fixture { + void set_program_options() { + options_description dummy; + _my.set_program_options(dummy, _cfg); + } + + void initialize(const std::vector& args){ + // We only have at most 3 arguments. OK to hardcodied in test + // programs. + const char* argv[10]; + EOS_ASSERT(args.size() < 10, chain::plugin_exception, "number of arguments (${size}) must be less than 10", ("size", args.size())); + + // argv[0] is program name, no need to fill in + for (auto i=0U; i& arg) { + set_program_options(); + initialize(arg); + } + + void plugin_startup(const std::vector& dirs, int runTimeSecs=3) { + set_options({"--resource-monitor-interval-seconds=1"}); + + for (auto& dir: dirs) { + _my.monitor_directory(dir); + } + + _my.plugin_startup(); + std::this_thread::sleep_for( std::chrono::milliseconds(runTimeSecs*1000) ); + _my.plugin_shutdown(); + } + + resource_monitor_plugin _my; + options_description _cfg; +}; + +BOOST_AUTO_TEST_SUITE(resmon_plugin_tests) + BOOST_FIXTURE_TEST_CASE(intervalTooBig, resmon_fixture) + { + BOOST_REQUIRE_THROW(set_options({"--resource-monitor-interval-seconds=301"}), chain::plugin_config_exception); + } + + BOOST_FIXTURE_TEST_CASE(intervalTooSmall, resmon_fixture) + { + BOOST_REQUIRE_THROW(set_options({"--resource-monitor-interval-seconds=0"}), chain::plugin_config_exception); + } + + BOOST_FIXTURE_TEST_CASE(intervalLowBound, resmon_fixture) + { + BOOST_REQUIRE_NO_THROW(set_options({"--resource-monitor-interval-seconds=1"})); + } + + BOOST_FIXTURE_TEST_CASE(intervalMiddle, resmon_fixture) + { + BOOST_REQUIRE_NO_THROW(set_options({"--resource-monitor-interval-seconds=150"})); + } + + BOOST_FIXTURE_TEST_CASE(intervalHighBound, resmon_fixture) + { + BOOST_REQUIRE_NO_THROW(set_options({"--resource-monitor-interval-seconds=300"})); + } + + BOOST_FIXTURE_TEST_CASE(thresholdTooBig, resmon_fixture) + { + BOOST_REQUIRE_THROW(set_options({"--resource-monitor-space-threshold=100"}), chain::plugin_config_exception); + } + + BOOST_FIXTURE_TEST_CASE(thresholdTooSmall, resmon_fixture) + { + BOOST_REQUIRE_THROW(set_options({"--resource-monitor-space-threshold=5"}), chain::plugin_config_exception); + } + + BOOST_FIXTURE_TEST_CASE(thresholdLowBound, resmon_fixture) + { + BOOST_REQUIRE_NO_THROW(set_options({"--resource-monitor-space-threshold=6"})); + } + + BOOST_FIXTURE_TEST_CASE(thresholdMiddle, resmon_fixture) + { + BOOST_REQUIRE_NO_THROW(set_options({"--resource-monitor-space-threshold=60"})); + } + + BOOST_FIXTURE_TEST_CASE(thresholdHighBound, resmon_fixture) + { + BOOST_REQUIRE_NO_THROW(set_options({"--resource-monitor-space-threshold=99"})); + } + + BOOST_FIXTURE_TEST_CASE(noShutdown, resmon_fixture) + { + BOOST_REQUIRE_NO_THROW(set_options({"--resource-monitor-not-shutdown-on-threshold-exceeded"})); + } + + BOOST_FIXTURE_TEST_CASE(startupNormal, resmon_fixture) + { + BOOST_REQUIRE_NO_THROW( plugin_startup({"/tmp"})); + } + + BOOST_FIXTURE_TEST_CASE(startupDuplicateDirs, resmon_fixture) + { + BOOST_REQUIRE_NO_THROW( plugin_startup({"/tmp", "/tmp"})); + } + + BOOST_FIXTURE_TEST_CASE(startupMultDirs, resmon_fixture) + { + // Under "/" are multiple file systems + BOOST_REQUIRE_NO_THROW( plugin_startup({"/", "/tmp"})); + } + + BOOST_FIXTURE_TEST_CASE(startupNoExistingDirs, resmon_fixture) + { + // "hsdfgd983" a random file and not existing + BOOST_REQUIRE_THROW( plugin_startup({"/tmp", "hsdfgd983"}), chain::plugin_config_exception); + } + + BOOST_FIXTURE_TEST_CASE(startupLongRun, resmon_fixture) + { + BOOST_REQUIRE_NO_THROW( plugin_startup({"/tmp"}, 120)); + } + + BOOST_FIXTURE_TEST_CASE(warningIntervalTooBig, resmon_fixture) + { + BOOST_REQUIRE_THROW(set_options({"--resource-monitor-warning-interval=451"}), chain::plugin_config_exception); + } + + BOOST_FIXTURE_TEST_CASE(warningIntervalTooSmall, resmon_fixture) + { + BOOST_REQUIRE_THROW(set_options({"--resource-monitor-warning-interval=0"}), chain::plugin_config_exception); + } + + BOOST_FIXTURE_TEST_CASE(warningIntervalLowBound, resmon_fixture) + { + BOOST_REQUIRE_NO_THROW(set_options({"--resource-monitor-warning-interval=1"})); + } + + BOOST_FIXTURE_TEST_CASE(warningIntervalMiddle, resmon_fixture) + { + BOOST_REQUIRE_NO_THROW(set_options({"--resource-monitor-warning-interval=225"})); + } + + BOOST_FIXTURE_TEST_CASE(warningIntervalHighBound, resmon_fixture) + { + BOOST_REQUIRE_NO_THROW(set_options({"--resource-monitor-warning-interval=450"})); + } + +BOOST_AUTO_TEST_SUITE_END() diff --git a/plugins/resource_monitor_plugin/test/test_threshold.cpp b/plugins/resource_monitor_plugin/test/test_threshold.cpp new file mode 100644 index 00000000000..afecd17bca7 --- /dev/null +++ b/plugins/resource_monitor_plugin/test/test_threshold.cpp @@ -0,0 +1,301 @@ +#define BOOST_TEST_MODULE threshold +#include + +#include + +#include + +using namespace eosio; +using namespace eosio::resource_monitor; +using namespace boost::system; + +struct threshold_fixture { + struct mock_space_provider { + mock_space_provider(threshold_fixture& fixture) + :fixture(fixture) + {} + + int get_stat(const char *path, struct stat *buf) const { + return fixture.mock_get_stat(path, buf); + } + + bfs::space_info get_space(const bfs::path& p, boost::system::error_code& ec) const { + return fixture.mock_get_space(p, ec); + } + + threshold_fixture& fixture; + }; + + boost::asio::io_context ctx; + + using file_space_handler_t = file_space_handler; + threshold_fixture() + : space_handler(mock_space_provider(*this), ctx) + { + } + + void add_file_system(const bfs::path& path_name) { + space_handler.add_file_system(path_name); + } + + void set_threshold(uint32_t threshold, uint32_t warning_threshold) { + space_handler.set_threshold(threshold, warning_threshold); + } + + bool is_threshold_exceeded() { + return space_handler.is_threshold_exceeded(); + } + + void set_shutdown_on_exceeded(bool shutdown_on_exceeded) { + space_handler.set_shutdown_on_exceeded(shutdown_on_exceeded); + } + + bool test_threshold_common(std::map& available, std::map& dev, uint32_t warning_threshold=75) + { + mock_get_space = [available]( const bfs::path& p, boost::system::error_code& ec) mutable -> bfs::space_info { + ec = boost::system::errc::make_error_code(errc::success); + + bfs::space_info rc; + rc.capacity = 1000000; + rc.available = available[p]; + + return rc; + }; + + mock_get_stat = [dev]( const char *path, struct stat *buf ) mutable -> int { + bfs::path name = path; + buf->st_dev = dev[name]; + + return 0; + }; + + set_threshold(80, warning_threshold); + set_shutdown_on_exceeded(true); + + for (auto i = 0U; i < available.size(); i++) { + add_file_system("/test" + std::to_string(i)); + } + + return is_threshold_exceeded(); + } + + // fixture data and methods + std::function mock_get_space; + std::function mock_get_stat; + + file_space_handler_t space_handler; +}; + +BOOST_AUTO_TEST_SUITE(threshol_tests) + BOOST_FIXTURE_TEST_CASE(equal_to_threshold, threshold_fixture) + { + std::map availables {{"/test0", 200000}}; + std::map devs {{"/test0", 0}}; + + BOOST_TEST( !test_threshold_common(availables, devs) ); + } + + BOOST_FIXTURE_TEST_CASE(above_threshold_1_byte, threshold_fixture) + { + std::map availables {{"/test0", 199999}}; + std::map devs {{"/test0", 0}}; + + BOOST_TEST( test_threshold_common(availables, devs) ); + } + + BOOST_FIXTURE_TEST_CASE(above_threshold_1000_byte, threshold_fixture) + { + std::map availables {{"/test0", 199000}}; + std::map devs {{"/test0", 0}}; + + BOOST_TEST( test_threshold_common(availables, devs) ); + } + + BOOST_FIXTURE_TEST_CASE(within_warning, threshold_fixture) + { + std::map availables {{"/test0", 249999}}; + std::map devs {{"/test0", 0}}; + + BOOST_TEST( !test_threshold_common(availables, devs) ); + } + + BOOST_FIXTURE_TEST_CASE(not_yet_warning, threshold_fixture) + { + std::map availables {{"/test0", 250001}}; + std::map devs {{"/test0", 0}}; + + BOOST_TEST( !test_threshold_common(availables, devs) ); + } + + BOOST_FIXTURE_TEST_CASE(below_threshold_1_byte, threshold_fixture) + { + std::map availables {{"/test0", 200001}}; + std::map devs {{"/test0", 0}}; + + BOOST_TEST( !test_threshold_common(availables, devs) ); + } + + BOOST_FIXTURE_TEST_CASE(below_threshold_500_byte, threshold_fixture) + { + std::map availables {{"/test0", 200500}}; + std::map devs {{"/test0", 0}}; + + BOOST_TEST( !test_threshold_common(availables, devs) ); + } + + BOOST_FIXTURE_TEST_CASE(first_file_system_over_threshold, threshold_fixture) + { + std::map availables {{"/test0", 199999}, + {"/test1", 200500}}; + std::map devs {{"/test0", 0}, + {"/test1", 1}}; + + BOOST_TEST( test_threshold_common(availables, devs) ); + } + + BOOST_FIXTURE_TEST_CASE(second_file_system_over_threshold, threshold_fixture) + { + std::map availables {{"/test0", 300000}, + {"/test1", 100000}}; + std::map devs {{"/test0", 0}, + {"/test1", 1}}; + + BOOST_TEST( test_threshold_common(availables, devs) ); + } + + BOOST_FIXTURE_TEST_CASE(no_file_system_over_threshold, threshold_fixture) + { + std::map availables {{"/test0", 300000}, + {"/test1", 200000}}; + std::map devs {{"/test0", 0}, + {"/test1", 1}}; + + BOOST_TEST( !test_threshold_common(availables, devs) ); + } + + BOOST_FIXTURE_TEST_CASE(both_file_systems_over_threshold, threshold_fixture) + { + std::map availables {{"/test0", 150000}, + {"/test1", 100000}}; + std::map devs {{"/test0", 0}, + {"/test1", 1}}; + + BOOST_TEST( test_threshold_common(availables, devs) ); + } + + BOOST_FIXTURE_TEST_CASE(one_of_three_over_threshold, threshold_fixture) + { + std::map availables {{"/test0", 300000}, + {"/test1", 199999}, + {"/test2", 250000}}; + std::map devs {{"/test0", 0}, + {"/test1", 1}, + {"/test2", 2}}; + + BOOST_TEST( test_threshold_common(availables, devs) ); + } + + BOOST_FIXTURE_TEST_CASE(one_of_three_over_threshold_dup, threshold_fixture) + { + std::map availables {{"/test0", 100000}, + {"/test1", 250000}, + {"/test2", 250000}}; + std::map devs {{"/test0", 0}, + {"/test1", 1}, // dup + {"/test2", 1}}; // dup + + BOOST_TEST( test_threshold_common(availables, devs) ); + } + + BOOST_FIXTURE_TEST_CASE(none_of_three_over_threshold, threshold_fixture) + { + std::map availables {{"/test0", 300000}, + {"/test1", 200000}, + {"/test2", 250000}}; + std::map devs {{"/test0", 0}, + {"/test1", 1}, + {"/test2", 2}}; + + BOOST_TEST( !test_threshold_common(availables, devs) ); + } + + BOOST_FIXTURE_TEST_CASE(none_of_three_over_threshold_dup, threshold_fixture) + { + std::map availables {{"/test0", 800000}, + {"/test1", 550000}, + {"/test2", 550000}}; + std::map devs {{"/test0", 0}, + {"/test1", 1}, // dup + {"/test2", 1}}; // dup + + BOOST_TEST( !test_threshold_common(availables, devs) ); + } + + BOOST_FIXTURE_TEST_CASE(warning_threshold_equal_to_threshold, threshold_fixture) + { + std::map availables {{"/test0", 150000}}; + std::map devs {{"/test0", 0}}; + + BOOST_REQUIRE_THROW(test_threshold_common(availables, devs, 80), chain::plugin_config_exception); + } + + BOOST_FIXTURE_TEST_CASE(warning_threshold_greater_than_threshold, threshold_fixture) + { + std::map availables {{"/test0", 150000}}; + std::map devs {{"/test0", 0}}; + + BOOST_REQUIRE_THROW( test_threshold_common(availables, devs, 85), chain::plugin_config_exception ); + } + + BOOST_FIXTURE_TEST_CASE(warning_threshold_less_than_threshold, threshold_fixture) + { + std::map availables {{"/test0", 200000}}; + std::map devs {{"/test0", 0}}; + + BOOST_TEST( !test_threshold_common(availables, devs, 70) ); + } + + BOOST_FIXTURE_TEST_CASE(get_space_failure_in_middle, threshold_fixture) + { + mock_get_space = [ i = 0 ]( const bfs::path& p, boost::system::error_code& ec) mutable -> bfs::space_info { + if ( i == 3 ) { + ec = boost::system::errc::make_error_code(errc::no_such_file_or_directory); + } else { + ec = boost::system::errc::make_error_code(errc::success); + } + + bfs::space_info rc; + rc.capacity = 1000000; + rc.available = 200500; + + i++; + + return rc; + }; + + mock_get_stat = []( const char *path, struct stat *buf ) -> int { + buf->st_dev = 0; + return 0; + }; + + set_threshold(80, 75); + add_file_system("/test"); + + auto expected_response = false; + + auto actual_response_0 = is_threshold_exceeded(); + auto actual_response_1 = is_threshold_exceeded(); + auto actual_response_2 = is_threshold_exceeded(); + auto actual_response_3 = is_threshold_exceeded(); + auto actual_response_4 = is_threshold_exceeded(); + auto actual_response_5 = is_threshold_exceeded(); + + BOOST_TEST(expected_response == actual_response_0); + BOOST_TEST(expected_response == actual_response_1); + BOOST_TEST(expected_response == actual_response_2); + BOOST_TEST(expected_response == actual_response_3); + BOOST_TEST(expected_response == actual_response_4); + BOOST_TEST(expected_response == actual_response_5); + } + +BOOST_AUTO_TEST_SUITE_END() diff --git a/plugins/signature_provider_plugin/CMakeLists.txt b/plugins/signature_provider_plugin/CMakeLists.txt new file mode 100644 index 00000000000..71447d6b2bc --- /dev/null +++ b/plugins/signature_provider_plugin/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB HEADERS "include/eosio/signature_provider_plugin/*.hpp") +add_library( signature_provider_plugin + signature_provider_plugin.cpp + ${HEADERS} ) + +target_link_libraries( signature_provider_plugin appbase fc http_client_plugin ) +target_include_directories( signature_provider_plugin PUBLIC include ) +if(APPLE) + target_link_libraries( signature_provider_plugin se-helpers ) +endif() diff --git a/plugins/signature_provider_plugin/include/eosio/signature_provider_plugin/signature_provider_plugin.hpp b/plugins/signature_provider_plugin/include/eosio/signature_provider_plugin/signature_provider_plugin.hpp new file mode 100644 index 00000000000..79118eba9ca --- /dev/null +++ b/plugins/signature_provider_plugin/include/eosio/signature_provider_plugin/signature_provider_plugin.hpp @@ -0,0 +1,33 @@ +#pragma once +#include +#include +#include + +namespace eosio { + +using namespace appbase; + +class signature_provider_plugin : public appbase::plugin { +public: + signature_provider_plugin(); + virtual ~signature_provider_plugin(); + + APPBASE_PLUGIN_REQUIRES((http_client_plugin)) + virtual void set_program_options(options_description&, options_description& cfg) override; + + void plugin_initialize(const variables_map& options); + void plugin_startup() {} + void plugin_shutdown() {} + + const char* const signature_provider_help_text() const; + + using signature_provider_type = std::function; + + std::pair signature_provider_for_specification(const std::string& spec) const; + signature_provider_type signature_provider_for_private_key(const chain::private_key_type priv) const; + +private: + std::unique_ptr my; +}; + +} diff --git a/plugins/signature_provider_plugin/signature_provider_plugin.cpp b/plugins/signature_provider_plugin/signature_provider_plugin.cpp new file mode 100644 index 00000000000..7dfa3dab647 --- /dev/null +++ b/plugins/signature_provider_plugin/signature_provider_plugin.cpp @@ -0,0 +1,125 @@ +#include +#include + +#include +#include + +#include + +#ifdef __APPLE__ +#include +#endif + +namespace eosio { + static appbase::abstract_plugin& _signature_provider_plugin = app().register_plugin(); + +class signature_provider_plugin_impl { + public: + fc::microseconds _keosd_provider_timeout_us; + + signature_provider_plugin::signature_provider_type + make_key_signature_provider(const chain::private_key_type& key) const { + return [key]( const chain::digest_type& digest ) { + return key.sign(digest); + }; + } + +#ifdef __APPLE__ + signature_provider_plugin::signature_provider_type + make_se_signature_provider(const chain::public_key_type pubkey) const { + EOS_ASSERT(secure_enclave::hardware_supports_secure_enclave(), chain::secure_enclave_exception, "Secure Enclave not supported on this hardware"); + EOS_ASSERT(secure_enclave::application_signed(), chain::secure_enclave_exception, "Application is not signed, Secure Enclave use not supported"); + + std::set allkeys = secure_enclave::get_all_keys(); + for(const auto& se_key : secure_enclave::get_all_keys()) + if(se_key.public_key() == pubkey) + return [se_key](const chain::digest_type& digest) { + return se_key.sign(digest); + }; + + EOS_THROW(chain::secure_enclave_exception, "${k} not found in Secure Enclave", ("k", pubkey)); + } +#endif + + signature_provider_plugin::signature_provider_type + make_keosd_signature_provider(const string& url_str, const chain::public_key_type pubkey) const { + fc::url keosd_url; + if(boost::algorithm::starts_with(url_str, "unix://")) + //send the entire string after unix:// to http_plugin. It'll auto-detect which part + // is the unix socket path, and which part is the url to hit on the server + keosd_url = fc::url("unix", url_str.substr(7), fc::ostring(), fc::ostring(), fc::ostring(), fc::ostring(), fc::ovariant_object(), std::optional()); + else + keosd_url = fc::url(url_str); + + return [to=_keosd_provider_timeout_us, keosd_url, pubkey](const chain::digest_type& digest) { + fc::variant params; + fc::to_variant(std::make_pair(digest, pubkey), params); + auto deadline = to.count() >= 0 ? fc::time_point::now() + to : fc::time_point::maximum(); + return app().get_plugin().get_client().post_sync(keosd_url, params, deadline).as(); + }; + } +}; + +signature_provider_plugin::signature_provider_plugin():my(new signature_provider_plugin_impl()){} +signature_provider_plugin::~signature_provider_plugin(){} + +void signature_provider_plugin::set_program_options(options_description&, options_description& cfg) { + cfg.add_options() + ("keosd-provider-timeout", boost::program_options::value()->default_value(5), + "Limits the maximum time (in milliseconds) that is allowed for sending requests to a keosd provider for signing") + ; +} + +const char* const signature_provider_plugin::signature_provider_help_text() const { + return "Key=Value pairs in the form =\n" + "Where:\n" + " \tis a string form of a vaild EOSIO public key\n\n" + " \tis a string in the form :\n\n" + " \tis KEY, KEOSD, or SE\n\n" + " KEY: \tis a string form of a valid EOSIO private key which maps to the provided public key\n\n" + " KEOSD: \tis the URL where keosd is available and the approptiate wallet(s) are unlocked\n\n" +#ifdef __APPLE__ + " SE: \tindicates the key resides in Secure Enclave" +#endif + ; + +} + +void signature_provider_plugin::plugin_initialize(const variables_map& options) { + my->_keosd_provider_timeout_us = fc::milliseconds( options.at("keosd-provider-timeout").as() ); +} + +std::pair +signature_provider_plugin::signature_provider_for_specification(const std::string& spec) const { + auto delim = spec.find("="); + EOS_ASSERT(delim != std::string::npos, chain::plugin_config_exception, "Missing \"=\" in the key spec pair"); + auto pub_key_str = spec.substr(0, delim); + auto spec_str = spec.substr(delim + 1); + + auto spec_delim = spec_str.find(":"); + EOS_ASSERT(spec_delim != std::string::npos, chain::plugin_config_exception, "Missing \":\" in the key spec pair"); + auto spec_type_str = spec_str.substr(0, spec_delim); + auto spec_data = spec_str.substr(spec_delim + 1); + + auto pubkey = chain::public_key_type(pub_key_str); + + if(spec_type_str == "KEY") { + chain::private_key_type priv(spec_data); + EOS_ASSERT(pubkey == priv.get_public_key(), chain::plugin_config_exception, "Private key does not match given public key for ${pub}", ("pub", pubkey)); + return std::make_pair(pubkey, my->make_key_signature_provider(priv)); + } + else if(spec_type_str == "KEOSD") + return std::make_pair(pubkey, my->make_keosd_signature_provider(spec_data, pubkey)); +#ifdef __APPLE__ + else if(spec_type_str == "SE") + return std::make_pair(pubkey, my->make_se_signature_provider(pubkey)); +#endif + EOS_THROW(chain::plugin_config_exception, "Unsupported key provider type \"${t}\"", ("t", spec_type_str)); +} + +signature_provider_plugin::signature_provider_type +signature_provider_plugin::signature_provider_for_private_key(const chain::private_key_type priv) const { + return my->make_key_signature_provider(priv); +} + +} diff --git a/plugins/state_history_plugin/CMakeLists.txt b/plugins/state_history_plugin/CMakeLists.txt index 21f0e10a900..8a72f9d12f3 100644 --- a/plugins/state_history_plugin/CMakeLists.txt +++ b/plugins/state_history_plugin/CMakeLists.txt @@ -1,8 +1,7 @@ file(GLOB HEADERS "include/eosio/state_history_plugin/*.hpp") add_library( state_history_plugin state_history_plugin.cpp - state_history_plugin_abi.cpp ${HEADERS} ) -target_link_libraries( state_history_plugin chain_plugin eosio_chain appbase ) +target_link_libraries( state_history_plugin state_history chain_plugin eosio_chain appbase ) target_include_directories( state_history_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_log.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_log.hpp deleted file mode 100644 index 3bf19eb012a..00000000000 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_log.hpp +++ /dev/null @@ -1,297 +0,0 @@ -#pragma once - -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace eosio { - -/* - * *.log: - * +---------+----------------+-----------+------------------+-----+---------+----------------+ - * | Entry i | Pos of Entry i | Entry i+1 | Pos of Entry i+1 | ... | Entry z | Pos of Entry z | - * +---------+----------------+-----------+------------------+-----+---------+----------------+ - * - * *.index: - * +----------------+------------------+-----+----------------+ - * | Pos of Entry i | Pos of Entry i+1 | ... | Pos of Entry z | - * +----------------+------------------+-----+----------------+ - * - * each entry: - * state_history_log_header - * payload - */ - -inline uint64_t ship_magic(uint32_t version) { return N(ship).to_uint64_t() | version; } -inline bool is_ship(uint64_t magic) { return (magic & 0xffff'ffff'0000'0000) == N(ship).to_uint64_t(); } -inline uint32_t get_ship_version(uint64_t magic) { return magic; } -inline bool is_ship_supported_version(uint64_t magic) { return get_ship_version(magic) == 0; } -static const uint32_t ship_current_version = 0; - -struct state_history_log_header { - uint64_t magic = ship_magic(ship_current_version); - chain::block_id_type block_id = {}; - uint64_t payload_size = 0; -}; -static const int state_history_log_header_serial_size = sizeof(state_history_log_header::magic) + - sizeof(state_history_log_header::block_id) + - sizeof(state_history_log_header::payload_size); - -class state_history_log { - private: - const char* const name = ""; - std::string log_filename; - std::string index_filename; - fc::cfile log; - fc::cfile index; - uint32_t _begin_block = 0; - uint32_t _end_block = 0; - chain::block_id_type last_block_id; - - public: - state_history_log(const char* const name, std::string log_filename, std::string index_filename) - : name(name) - , log_filename(std::move(log_filename)) - , index_filename(std::move(index_filename)) { - open_log(); - open_index(); - } - - uint32_t begin_block() const { return _begin_block; } - uint32_t end_block() const { return _end_block; } - - void read_header(state_history_log_header& header, bool assert_version = true) { - char bytes[state_history_log_header_serial_size]; - log.read(bytes, sizeof(bytes)); - fc::datastream ds(bytes, sizeof(bytes)); - fc::raw::unpack(ds, header); - EOS_ASSERT(!ds.remaining(), chain::plugin_exception, "state_history_log_header_serial_size mismatch"); - if (assert_version) - EOS_ASSERT(is_ship(header.magic) && is_ship_supported_version(header.magic), chain::plugin_exception, - "corrupt ${name}.log (0)", ("name", name)); - } - - void write_header(const state_history_log_header& header) { - char bytes[state_history_log_header_serial_size]; - fc::datastream ds(bytes, sizeof(bytes)); - fc::raw::pack(ds, header); - EOS_ASSERT(!ds.remaining(), chain::plugin_exception, "state_history_log_header_serial_size mismatch"); - log.write(bytes, sizeof(bytes)); - } - - template - void write_entry(const state_history_log_header& header, const chain::block_id_type& prev_id, F write_payload) { - auto block_num = chain::block_header::num_from_id(header.block_id); - EOS_ASSERT(_begin_block == _end_block || block_num <= _end_block, chain::plugin_exception, - "missed a block in ${name}.log", ("name", name)); - - if (_begin_block != _end_block && block_num > _begin_block) { - if (block_num == _end_block) { - EOS_ASSERT(prev_id == last_block_id, chain::plugin_exception, "missed a fork change in ${name}.log", - ("name", name)); - } else { - state_history_log_header prev; - get_entry(block_num - 1, prev); - EOS_ASSERT(prev_id == prev.block_id, chain::plugin_exception, "missed a fork change in ${name}.log", - ("name", name)); - } - } - - if (block_num < _end_block) - truncate(block_num); - log.seek_end(0); - uint64_t pos = log.tellp(); - write_header(header); - write_payload(log); - uint64_t end = log.tellp(); - EOS_ASSERT(end == pos + state_history_log_header_serial_size + header.payload_size, chain::plugin_exception, - "wrote payload with incorrect size to ${name}.log", ("name", name)); - log.write((char*)&pos, sizeof(pos)); - - index.seek_end(0); - index.write((char*)&pos, sizeof(pos)); - if (_begin_block == _end_block) - _begin_block = block_num; - _end_block = block_num + 1; - last_block_id = header.block_id; - } - - // returns cfile positioned at payload - fc::cfile& get_entry(uint32_t block_num, state_history_log_header& header) { - EOS_ASSERT(block_num >= _begin_block && block_num < _end_block, chain::plugin_exception, - "read non-existing block in ${name}.log", ("name", name)); - log.seek(get_pos(block_num)); - read_header(header); - return log; - } - - chain::block_id_type get_block_id(uint32_t block_num) { - state_history_log_header header; - get_entry(block_num, header); - return header.block_id; - } - - private: - bool get_last_block(uint64_t size) { - state_history_log_header header; - uint64_t suffix; - log.seek(size - sizeof(suffix)); - log.read((char*)&suffix, sizeof(suffix)); - if (suffix > size || suffix + state_history_log_header_serial_size > size) { - elog("corrupt ${name}.log (2)", ("name", name)); - return false; - } - log.seek(suffix); - read_header(header, false); - if (!is_ship(header.magic) || !is_ship_supported_version(header.magic) || - suffix + state_history_log_header_serial_size + header.payload_size + sizeof(suffix) != size) { - elog("corrupt ${name}.log (3)", ("name", name)); - return false; - } - _end_block = chain::block_header::num_from_id(header.block_id) + 1; - last_block_id = header.block_id; - if (_begin_block >= _end_block) { - elog("corrupt ${name}.log (4)", ("name", name)); - return false; - } - return true; - } - - void recover_blocks(uint64_t size) { - ilog("recover ${name}.log", ("name", name)); - uint64_t pos = 0; - uint32_t num_found = 0; - while (true) { - state_history_log_header header; - if (pos + state_history_log_header_serial_size > size) - break; - log.seek(pos); - read_header(header, false); - uint64_t suffix; - if (!is_ship(header.magic) || !is_ship_supported_version(header.magic) || header.payload_size > size || - pos + state_history_log_header_serial_size + header.payload_size + sizeof(suffix) > size) { - EOS_ASSERT(!is_ship(header.magic) || is_ship_supported_version(header.magic), chain::plugin_exception, - "${name}.log has an unsupported version", ("name", name)); - break; - } - log.seek(pos + state_history_log_header_serial_size + header.payload_size); - log.read((char*)&suffix, sizeof(suffix)); - if (suffix != pos) - break; - pos = pos + state_history_log_header_serial_size + header.payload_size + sizeof(suffix); - if (!(++num_found % 10000)) { - printf("%10u blocks found, log pos=%12llu\r", (unsigned)num_found, (unsigned long long)pos); - fflush(stdout); - } - } - log.flush(); - boost::filesystem::resize_file(log_filename, pos); - log.flush(); - EOS_ASSERT(get_last_block(pos), chain::plugin_exception, "recover ${name}.log failed", ("name", name)); - } - - void open_log() { - log.set_file_path( log_filename ); - log.open( "a+b" ); // std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app - log.seek_end(0); - uint64_t size = log.tellp(); - if (size >= state_history_log_header_serial_size) { - state_history_log_header header; - log.seek(0); - read_header(header, false); - EOS_ASSERT(is_ship(header.magic) && is_ship_supported_version(header.magic) && - state_history_log_header_serial_size + header.payload_size + sizeof(uint64_t) <= size, - chain::plugin_exception, "corrupt ${name}.log (1)", ("name", name)); - _begin_block = chain::block_header::num_from_id(header.block_id); - last_block_id = header.block_id; - if (!get_last_block(size)) - recover_blocks(size); - ilog("${name}.log has blocks ${b}-${e}", ("name", name)("b", _begin_block)("e", _end_block - 1)); - } else { - EOS_ASSERT(!size, chain::plugin_exception, "corrupt ${name}.log (5)", ("name", name)); - ilog("${name}.log is empty", ("name", name)); - } - } - - void open_index() { - index.set_file_path( index_filename ); - index.open( "a+b" ); // std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app - index.seek_end(0); - if (index.tellp() == (static_cast(_end_block) - _begin_block) * sizeof(uint64_t)) - return; - ilog("Regenerate ${name}.index", ("name", name)); - index.close(); - index.open( "w+b" ); // std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::trunc - - log.seek_end(0); - uint64_t size = log.tellp(); - uint64_t pos = 0; - uint32_t num_found = 0; - while (pos < size) { - state_history_log_header header; - EOS_ASSERT(pos + state_history_log_header_serial_size <= size, chain::plugin_exception, - "corrupt ${name}.log (6)", ("name", name)); - log.seek(pos); - read_header(header, false); - uint64_t suffix_pos = pos + state_history_log_header_serial_size + header.payload_size; - uint64_t suffix; - EOS_ASSERT(is_ship(header.magic) && is_ship_supported_version(header.magic) && - suffix_pos + sizeof(suffix) <= size, - chain::plugin_exception, "corrupt ${name}.log (7)", ("name", name)); - log.seek(suffix_pos); - log.read((char*)&suffix, sizeof(suffix)); - // ilog("block ${b} at ${pos}-${end} suffix=${suffix} file_size=${fs}", - // ("b", header.block_num)("pos", pos)("end", suffix_pos + sizeof(suffix))("suffix", suffix)("fs", size)); - EOS_ASSERT(suffix == pos, chain::plugin_exception, "corrupt ${name}.log (8)", ("name", name)); - - index.write((char*)&pos, sizeof(pos)); - pos = suffix_pos + sizeof(suffix); - if (!(++num_found % 10000)) { - printf("%10u blocks found, log pos=%12llu\r", (unsigned)num_found, (unsigned long long)pos); - fflush(stdout); - } - } - } - - uint64_t get_pos(uint32_t block_num) { - uint64_t pos; - index.seek((block_num - _begin_block) * sizeof(pos)); - index.read((char*)&pos, sizeof(pos)); - return pos; - } - - void truncate(uint32_t block_num) { - log.flush(); - index.flush(); - uint64_t num_removed = 0; - if (block_num <= _begin_block) { - num_removed = _end_block - _begin_block; - log.seek(0); - index.seek(0); - boost::filesystem::resize_file(log_filename, 0); - boost::filesystem::resize_file(index_filename, 0); - _begin_block = _end_block = 0; - } else { - num_removed = _end_block - block_num; - uint64_t pos = get_pos(block_num); - log.seek(0); - index.seek(0); - boost::filesystem::resize_file(log_filename, pos); - boost::filesystem::resize_file(index_filename, (block_num - _begin_block) * sizeof(uint64_t)); - _end_block = block_num; - } - log.flush(); - index.flush(); - ilog("fork or replay: removed ${n} blocks from ${name}.log", ("n", num_removed)("name", name)); - } -}; // state_history_log - -} // namespace eosio - -FC_REFLECT(eosio::state_history_log_header, (magic)(block_id)(payload_size)) diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp index 66f8de30b53..a9e4c021dad 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp @@ -2,11 +2,7 @@ #include #include - -template -struct history_serial_big_vector_wrapper { - T obj; -}; +#include namespace fc { class variant; @@ -18,103 +14,6 @@ using std::shared_ptr; typedef shared_ptr state_history_ptr; -struct partial_transaction { - chain::time_point_sec expiration = {}; - uint16_t ref_block_num = {}; - uint32_t ref_block_prefix = {}; - fc::unsigned_int max_net_usage_words = {}; - uint8_t max_cpu_usage_ms = {}; - fc::unsigned_int delay_sec = {}; - chain::extensions_type transaction_extensions = {}; - vector signatures = {}; - vector context_free_data = {}; - - partial_transaction(const chain::signed_transaction& t) - : expiration(t.expiration) - , ref_block_num(t.ref_block_num) - , ref_block_prefix(t.ref_block_prefix) - , max_net_usage_words(t.max_net_usage_words) - , max_cpu_usage_ms(t.max_cpu_usage_ms) - , delay_sec(t.delay_sec) - , transaction_extensions(t.transaction_extensions) - , signatures(t.signatures) - , context_free_data(t.context_free_data) {} -}; - -struct augmented_transaction_trace { - chain::transaction_trace_ptr trace; - std::shared_ptr partial; - - augmented_transaction_trace() = default; - augmented_transaction_trace(const augmented_transaction_trace&) = default; - augmented_transaction_trace(augmented_transaction_trace&&) = default; - - augmented_transaction_trace(const chain::transaction_trace_ptr& trace) - : trace{trace} {} - - augmented_transaction_trace(const chain::transaction_trace_ptr& trace, - const std::shared_ptr& partial) - : trace{trace} - , partial{partial} {} - - augmented_transaction_trace(const chain::transaction_trace_ptr& trace, const chain::signed_transaction& t) - : trace{trace} - , partial{std::make_shared(t)} {} - - augmented_transaction_trace& operator=(const augmented_transaction_trace&) = default; - augmented_transaction_trace& operator=(augmented_transaction_trace&&) = default; -}; - -struct table_delta { - fc::unsigned_int struct_version = 0; - std::string name{}; - history_serial_big_vector_wrapper>> rows{}; -}; - -struct block_position { - uint32_t block_num = 0; - chain::block_id_type block_id = {}; -}; - -struct get_status_request_v0 {}; - -struct get_status_result_v0 { - block_position head = {}; - block_position last_irreversible = {}; - uint32_t trace_begin_block = 0; - uint32_t trace_end_block = 0; - uint32_t chain_state_begin_block = 0; - uint32_t chain_state_end_block = 0; -}; - -struct get_blocks_request_v0 { - uint32_t start_block_num = 0; - uint32_t end_block_num = 0; - uint32_t max_messages_in_flight = 0; - std::vector have_positions = {}; - bool irreversible_only = false; - bool fetch_block = false; - bool fetch_traces = false; - bool fetch_deltas = false; -}; - -struct get_blocks_ack_request_v0 { - uint32_t num_messages = 0; -}; - -struct get_blocks_result_v0 { - block_position head; - block_position last_irreversible; - fc::optional this_block; - fc::optional prev_block; - fc::optional block; - fc::optional traces; - fc::optional deltas; -}; - -using state_request = fc::static_variant; -using state_result = fc::static_variant; - class state_history_plugin : public plugin { public: APPBASE_PLUGIN_REQUIRES((chain_plugin)) @@ -128,17 +27,10 @@ class state_history_plugin : public plugin { void plugin_startup(); void plugin_shutdown(); + void handle_sighup() override; + private: state_history_ptr my; }; } // namespace eosio - -// clang-format off -FC_REFLECT(eosio::table_delta, (struct_version)(name)(rows)); -FC_REFLECT(eosio::block_position, (block_num)(block_id)); -FC_REFLECT_EMPTY(eosio::get_status_request_v0); -FC_REFLECT(eosio::get_status_result_v0, (head)(last_irreversible)(trace_begin_block)(trace_end_block)(chain_state_begin_block)(chain_state_end_block)); -FC_REFLECT(eosio::get_blocks_request_v0, (start_block_num)(end_block_num)(max_messages_in_flight)(have_positions)(irreversible_only)(fetch_block)(fetch_traces)(fetch_deltas)); -FC_REFLECT(eosio::get_blocks_ack_request_v0, (num_messages)); -// clang-format on diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 3c892b2f041..2c3891903e6 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -1,6 +1,10 @@ #include -#include -#include +#include +#include +#include +#include + +#include #include #include @@ -8,9 +12,6 @@ #include #include #include -#include -#include -#include #include using tcp = boost::asio::ip::tcp; @@ -20,163 +21,74 @@ extern const char* const state_history_plugin_abi; namespace eosio { using namespace chain; +using namespace state_history; using boost::signals2::scoped_connection; static appbase::abstract_plugin& _state_history_plugin = app().register_plugin(); +const std::string logger_name("state_history"); +fc::logger _log; + template auto catch_and_log(F f) { try { return f(); } catch (const fc::exception& e) { - elog("${e}", ("e", e.to_detail_string())); + fc_elog(_log, "${e}", ("e", e.to_detail_string())); } catch (const std::exception& e) { - elog("${e}", ("e", e.what())); + fc_elog(_log, "${e}", ("e", e.what())); } catch (...) { - elog("unknown exception"); + fc_elog(_log, "unknown exception"); } } -namespace bio = boost::iostreams; -static bytes zlib_compress_bytes(bytes in) { - bytes out; - bio::filtering_ostream comp; - comp.push(bio::zlib_compressor(bio::zlib::default_compression)); - comp.push(bio::back_inserter(out)); - bio::write(comp, in.data(), in.size()); - bio::close(comp); - return out; -} - -static bytes zlib_decompress(const bytes& in) { - bytes out; - bio::filtering_ostream decomp; - decomp.push(bio::zlib_decompressor()); - decomp.push(bio::back_inserter(out)); - bio::write(decomp, in.data(), in.size()); - bio::close(decomp); - return out; -} - -template -bool include_delta(const T& old, const T& curr) { - return true; -} - -bool include_delta(const eosio::chain::table_id_object& old, const eosio::chain::table_id_object& curr) { - return old.payer != curr.payer; -} - -bool include_delta(const eosio::chain::resource_limits::resource_limits_object& old, - const eosio::chain::resource_limits::resource_limits_object& curr) { - return // - old.net_weight != curr.net_weight || // - old.cpu_weight != curr.cpu_weight || // - old.ram_bytes != curr.ram_bytes; -} - -bool include_delta(const eosio::chain::resource_limits::resource_limits_state_object& old, - const eosio::chain::resource_limits::resource_limits_state_object& curr) { - return // - old.average_block_net_usage.last_ordinal != curr.average_block_net_usage.last_ordinal || // - old.average_block_net_usage.value_ex != curr.average_block_net_usage.value_ex || // - old.average_block_net_usage.consumed != curr.average_block_net_usage.consumed || // - old.average_block_cpu_usage.last_ordinal != curr.average_block_cpu_usage.last_ordinal || // - old.average_block_cpu_usage.value_ex != curr.average_block_cpu_usage.value_ex || // - old.average_block_cpu_usage.consumed != curr.average_block_cpu_usage.consumed || // - old.total_net_weight != curr.total_net_weight || // - old.total_cpu_weight != curr.total_cpu_weight || // - old.total_ram_bytes != curr.total_ram_bytes || // - old.virtual_net_limit != curr.virtual_net_limit || // - old.virtual_cpu_limit != curr.virtual_cpu_limit; -} - -bool include_delta(const eosio::chain::account_metadata_object& old, - const eosio::chain::account_metadata_object& curr) { - return // - old.name != curr.name || // - old.is_privileged() != curr.is_privileged() || // - old.last_code_update != curr.last_code_update || // - old.vm_type != curr.vm_type || // - old.vm_version != curr.vm_version || // - old.code_hash != curr.code_hash; -} - -bool include_delta(const eosio::chain::code_object& old, const eosio::chain::code_object& curr) { // - return false; -} - -bool include_delta(const eosio::chain::protocol_state_object& old, const eosio::chain::protocol_state_object& curr) { - return old.activated_protocol_features != curr.activated_protocol_features; -} - struct state_history_plugin_impl : std::enable_shared_from_this { chain_plugin* chain_plug = nullptr; - fc::optional trace_log; - fc::optional chain_state_log; - bool trace_debug_mode = false; + std::optional trace_log; + std::optional chain_state_log; bool stopping = false; - fc::optional applied_transaction_connection; - fc::optional block_start_connection; - fc::optional accepted_block_connection; + std::optional applied_transaction_connection; + std::optional block_start_connection; + std::optional accepted_block_connection; string endpoint_address = "0.0.0.0"; uint16_t endpoint_port = 8080; std::unique_ptr acceptor; - std::map cached_traces; - fc::optional onblock_trace; - void get_log_entry(state_history_log& log, uint32_t block_num, fc::optional& result) { - if (block_num < log.begin_block() || block_num >= log.end_block()) - return; - state_history_log_header header; - auto& stream = log.get_entry(block_num, header); - uint32_t s; - stream.read((char*)&s, sizeof(s)); - bytes compressed(s); - if (s) - stream.read(compressed.data(), s); - result = zlib_decompress(compressed); - } + std::optional get_block_id(uint32_t block_num) { + std::optional result; - void get_block(uint32_t block_num, fc::optional& result) { - chain::signed_block_ptr p; - try { - p = chain_plug->chain().fetch_block_by_number(block_num); - } catch (...) { - return; - } - if (p) - result = fc::raw::pack(*p); - } + if (trace_log) + result = trace_log->get_block_id(block_num); + + if (!result && chain_state_log) + result = chain_state_log->get_block_id(block_num); + + if (result) + return result; - fc::optional get_block_id(uint32_t block_num) { - if (trace_log && block_num >= trace_log->begin_block() && block_num < trace_log->end_block()) - return trace_log->get_block_id(block_num); - if (chain_state_log && block_num >= chain_state_log->begin_block() && block_num < chain_state_log->end_block()) - return chain_state_log->get_block_id(block_num); try { - auto block = chain_plug->chain().fetch_block_by_number(block_num); - if (block) - return block->id(); + return chain_plug->chain().get_block_id_for_num(block_num); } catch (...) { + return {}; } - return {}; } + using get_blocks_request = std::variant; + struct session : std::enable_shared_from_this { std::shared_ptr plugin; std::unique_ptr> socket_stream; bool sending = false; bool sent_abi = false; std::vector> send_queue; - fc::optional current_request; + std::optional current_request; bool need_to_send_update = false; session(std::shared_ptr plugin) : plugin(std::move(plugin)) {} void start(tcp::socket socket) { - ilog("incoming connection"); + fc_ilog(_log, "incoming connection"); socket_stream = std::make_unique>(std::move(socket)); socket_stream->binary(true); socket_stream->next_layer().set_option(boost::asio::ip::tcp::no_delay(true)); @@ -200,7 +112,7 @@ struct state_history_plugin_impl : std::enable_shared_from_this ds(d, s); state_request req; fc::raw::unpack(ds, req); - req.visit(*self); + std::visit(*self, req); self->start_read(); }); }); @@ -238,10 +150,12 @@ struct state_history_plugin_impl : std::enable_shared_from_thischain_plug->chain(); get_status_result_v0 result; result.head = {chain.head_block_num(), chain.head_block_id()}; result.last_irreversible = {chain.last_irreversible_block_num(), chain.last_irreversible_block_id()}; + result.chain_id = chain.get_chain_id(); if (plugin->trace_log) { result.trace_begin_block = plugin->trace_log->begin_block(); result.trace_end_block = plugin->trace_log->end_block(); @@ -250,79 +164,164 @@ struct state_history_plugin_impl : std::enable_shared_from_thischain_state_log->begin_block(); result.chain_state_end_block = plugin->chain_state_log->end_block(); } + fc_ilog(_log, "pushing get_status_result_v0 to send queue"); send(std::move(result)); } - void operator()(get_blocks_request_v0& req) { + template + std::enable_if_t> + operator()(T& req) { + fc_ilog(_log, "received get_blocks_request = ${req}", ("req",req) ); for (auto& cp : req.have_positions) { if (req.start_block_num <= cp.block_num) continue; auto id = plugin->get_block_id(cp.block_num); if (!id || *id != cp.block_id) req.start_block_num = std::min(req.start_block_num, cp.block_num); + + if (!id) { + fc_dlog(_log, "block ${block_num} is not available", ("block_num", cp.block_num)); + } else if (*id != cp.block_id) { + fc_dlog(_log, "the id for block ${block_num} in block request have_positions does not match the existing", ("block_num", cp.block_num)); + } } req.have_positions.clear(); + fc_dlog(_log, " get_blocks_request start_block_num set to ${num}", ("num", req.start_block_num)); + current_request = req; + send_update(true); } - void operator()(get_blocks_ack_request_v0& req) { - if (!current_request) + void operator()(get_blocks_ack_request_v0& ack_req) { + fc_ilog(_log, "received get_blocks_ack_request_v0 = ${req}", ("req",ack_req)); + if (!current_request.has_value()) { + fc_dlog(_log, " no current get_blocks_request_v0, discarding the get_blocks_ack_request_v0"); return; - current_request->max_messages_in_flight += req.num_messages; + } + std::visit([num_messages = ack_req.num_messages](auto& req) { req.max_messages_in_flight += num_messages; }, + *current_request); + send_update(); } - void send_update(get_blocks_result_v0 result) { + void set_result_block_header(get_blocks_result_v1&, const signed_block_ptr& block) {} + void set_result_block_header(get_blocks_result_v2& result, const signed_block_ptr& block) { + bool fetch_block_header = std::get(*current_request).fetch_block_header; + if (fetch_block_header && block) { + result.block_header = static_cast(*block); + } + } + + uint32_t max_messages_in_flight() const { + if (current_request) + return std::visit( [](const auto& x){ return x.max_messages_in_flight; }, *current_request); + return 0; + } + + template + std::enable_if_t || std::is_same_v> + send_update(const block_state_ptr& head_block_state, T&& result) { need_to_send_update = true; - if (!send_queue.empty() || !current_request || !current_request->max_messages_in_flight) + if (!send_queue.empty() || !max_messages_in_flight() ) return; - auto& chain = plugin->chain_plug->chain(); + get_blocks_request_v0& block_req = std::visit([](auto& x) ->get_blocks_request_v0&{ return x; }, *current_request); + + auto& chain = plugin->chain_plug->chain(); result.last_irreversible = {chain.last_irreversible_block_num(), chain.last_irreversible_block_id()}; uint32_t current = - current_request->irreversible_only ? result.last_irreversible.block_num : result.head.block_num; - if (current_request->start_block_num <= current && - current_request->start_block_num < current_request->end_block_num) { - auto block_id = plugin->get_block_id(current_request->start_block_num); + block_req.irreversible_only ? result.last_irreversible.block_num : result.head.block_num; + if (block_req.start_block_num <= current && + block_req.start_block_num < block_req.end_block_num) { + + auto& block_num = block_req.start_block_num; + auto block_id = plugin->get_block_id(block_num); + + auto get_block = [&chain, block_num, head_block_state]() -> signed_block_ptr { + try { + if (head_block_state->block_num == block_num) + return head_block_state->block; + return chain.fetch_block_by_number(block_num); + } catch (...) { + return {}; + } + }; + if (block_id) { - result.this_block = block_position{current_request->start_block_num, *block_id}; - auto prev_block_id = plugin->get_block_id(current_request->start_block_num - 1); - if (prev_block_id) - result.prev_block = block_position{current_request->start_block_num - 1, *prev_block_id}; - if (current_request->fetch_block) - plugin->get_block(current_request->start_block_num, result.block); - if (current_request->fetch_traces && plugin->trace_log) - plugin->get_log_entry(*plugin->trace_log, current_request->start_block_num, result.traces); - if (current_request->fetch_deltas && plugin->chain_state_log) - plugin->get_log_entry(*plugin->chain_state_log, current_request->start_block_num, result.deltas); + result.this_block = block_position{block_num, *block_id}; + auto prev_block_id = plugin->get_block_id(block_num - 1); + if (prev_block_id) + result.prev_block = block_position{block_num - 1, *prev_block_id}; + if (block_req.fetch_block) { + result.block = signed_block_ptr_variant{get_block()}; + } + if (block_req.fetch_traces && plugin->trace_log) { + result.traces = plugin->trace_log->get_log_entry(block_num); + } + if (block_req.fetch_deltas && plugin->chain_state_log) { + result.deltas = plugin->chain_state_log->get_log_entry(block_num); + } + set_result_block_header(result, get_block()); } - ++current_request->start_block_num; + ++block_num; } + if (!result.has_value()) + return; + fc_ilog(_log, + "pushing result " + "{\"head\":{\"block_num\":${head}},\"last_irreversible\":{\"block_num\":${last_irr}},\"this_block\":{" + "\"block_num\":${this_block}}} to send queue", + ("head", result.head.block_num)("last_irr", result.last_irreversible.block_num)( + "this_block", result.this_block ? result.this_block->block_num : fc::variant())); send(std::move(result)); - --current_request->max_messages_in_flight; - need_to_send_update = current_request->start_block_num <= current && - current_request->start_block_num < current_request->end_block_num; + --block_req.max_messages_in_flight; + need_to_send_update = block_req.start_block_num <= current && + block_req.start_block_num < block_req.end_block_num; + + std::visit( []( auto&& ptr ) { + if( ptr ) { + if (fc::zipkin_config::is_enabled()) { + auto id = ptr->calculate_id(); + auto blk_trace = fc_create_trace_with_id( "Block", id ); + auto blk_span = fc_create_span( blk_trace, "SHiP-Send" ); + fc_add_tag( blk_span, "block_id", id ); + fc_add_tag( blk_span, "block_num", ptr->block_num() ); + fc_add_tag( blk_span, "block_time", ptr->timestamp.to_time_point() ); + } + } + }, result.block ); + } + + void send_update_for_block(const block_state_ptr& head_block_state) { + std::visit( + [&head_block_state, this](const auto& req) { + // send get_blocks_result_v1 when the request is get_blocks_request_v0 and + // send send_block_result_v2 when the request is get_blocks_request_v1. + if (head_block_state->block) { + typename std::decay_t::response_type result; + result.head = { head_block_state->block_num, head_block_state->id }; + send_update(head_block_state, std::move(result)); + } + }, + *current_request); } void send_update(const block_state_ptr& block_state) { need_to_send_update = true; - if (!send_queue.empty() || !current_request || !current_request->max_messages_in_flight) + if (!send_queue.empty() || !max_messages_in_flight()) return; - get_blocks_result_v0 result; - result.head = {block_state->block_num, block_state->id}; - send_update(std::move(result)); + + send_update_for_block(block_state); } void send_update(bool changed = false) { if (changed) need_to_send_update = true; - if (!send_queue.empty() || !need_to_send_update || !current_request || - !current_request->max_messages_in_flight) + if (!send_queue.empty() || !need_to_send_update || + !max_messages_in_flight()) return; auto& chain = plugin->chain_plug->chain(); - get_blocks_result_v0 result; - result.head = {chain.head_block_num(), chain.head_block_id()}; - send_update(std::move(result)); + send_update_for_block(chain.head_block_state()); } template @@ -330,13 +329,13 @@ struct state_history_plugin_impl : std::enable_shared_from_thisreceipt && trace_log) { - if (chain::is_onblock(*p)) - onblock_trace.emplace(p, t); - else if (p->failed_dtrx_trace) - cached_traces[p->failed_dtrx_trace->id] = augmented_transaction_trace{p, t}; - else - cached_traces[p->id] = augmented_transaction_trace{p, t}; + void on_applied_transaction(const transaction_trace_ptr& p, const packed_transaction_ptr& t) { + if (trace_log) + trace_log->add_transaction(p, t); + } + + void store(const block_state_ptr& block_state) { + try { + if (trace_log) + trace_log->store(chain_plug->chain().db(), block_state); + if (chain_state_log) + chain_state_log->store(chain_plug->chain().kv_db(), block_state); + return; } + FC_LOG_AND_DROP() + + // Both app().quit() and exception throwing are required. Without app().quit(), + // the exception would be caught and drop before reaching main(). The exception is + // to ensure the block won't be committed. + appbase::app().quit(); + EOS_THROW( + chain::state_history_write_exception, + "State history encountered an Error which it cannot recover from. Please resolve the error and relaunch " + "the process"); } void on_accepted_block(const block_state_ptr& block_state) { - store_traces(block_state); - store_chain_state(block_state); + auto blk_trace = fc_create_trace_with_id("Block", block_state->id); + auto blk_span = fc_create_span(blk_trace, "SHiP-Accepted"); + fc_add_tag(blk_span, "block_id", block_state->id); + fc_add_tag(blk_span, "block_num", block_state->block_num); + fc_add_tag(blk_span, "block_time", block_state->block->timestamp.to_time_point()); + this->store(block_state); for (auto& s : sessions) { auto& p = s.second; if (p) { - if (p->current_request && block_state->block_num < p->current_request->start_block_num) - p->current_request->start_block_num = block_state->block_num; + if (p->current_request) { + uint32_t& req_start_block_num = + std::visit([](auto& req) -> uint32_t& { return req.start_block_num; }, *p->current_request); + if (block_state->block_num < req_start_block_num) { + req_start_block_num = block_state->block_num; + } + } p->send_update(block_state); } } } void on_block_start(uint32_t block_num) { - clear_caches(); + if (trace_log) + trace_log->block_start(block_num); } - void clear_caches() { - cached_traces.clear(); - onblock_trace.reset(); - } - - void store_traces(const block_state_ptr& block_state) { - if (!trace_log) - return; - std::vector traces; - if (onblock_trace) - traces.push_back(*onblock_trace); - for (auto& r : block_state->block->transactions) { - transaction_id_type id; - if (r.trx.contains()) - id = r.trx.get(); - else - id = r.trx.get().id(); - auto it = cached_traces.find(id); - EOS_ASSERT(it != cached_traces.end() && it->second.trace->receipt, plugin_exception, - "missing trace for transaction ${id}", ("id", id)); - traces.push_back(it->second); - } - clear_caches(); - - auto& db = chain_plug->chain().db(); - auto traces_bin = zlib_compress_bytes(fc::raw::pack(make_history_context_wrapper(db, trace_debug_mode, traces))); - EOS_ASSERT(traces_bin.size() == (uint32_t)traces_bin.size(), plugin_exception, "traces is too big"); - - state_history_log_header header{.magic = ship_magic(ship_current_version), - .block_id = block_state->block->id(), - .payload_size = sizeof(uint32_t) + traces_bin.size()}; - trace_log->write_entry(header, block_state->block->previous, [&](auto& stream) { - uint32_t s = (uint32_t)traces_bin.size(); - stream.write((char*)&s, sizeof(s)); - if (!traces_bin.empty()) - stream.write(traces_bin.data(), traces_bin.size()); - }); - } - - void store_chain_state(const block_state_ptr& block_state) { - if (!chain_state_log) - return; - bool fresh = chain_state_log->begin_block() == chain_state_log->end_block(); - if (fresh) - ilog("Placing initial state in block ${n}", ("n", block_state->block->block_num())); - - std::vector deltas; - auto& db = chain_plug->chain().db(); - - const auto& table_id_index = db.get_index(); - std::map removed_table_id; - for (auto& rem : table_id_index.stack().back().removed_values) - removed_table_id[rem.first._id] = &rem.second; - - auto get_table_id = [&](uint64_t tid) -> const table_id_object& { - auto obj = table_id_index.find(tid); - if (obj) - return *obj; - auto it = removed_table_id.find(tid); - EOS_ASSERT(it != removed_table_id.end(), chain::plugin_exception, "can not found table id ${tid}", - ("tid", tid)); - return *it->second; - }; - - auto pack_row = [&](auto& row) { return fc::raw::pack(make_history_serial_wrapper(db, row)); }; - auto pack_contract_row = [&](auto& row) { - return fc::raw::pack(make_history_context_wrapper(db, get_table_id(row.t_id._id), row)); - }; - - auto process_table = [&](auto* name, auto& index, auto& pack_row) { - if (fresh) { - if (index.indices().empty()) - return; - deltas.push_back({}); - auto& delta = deltas.back(); - delta.name = name; - for (auto& row : index.indices()) - delta.rows.obj.emplace_back(true, pack_row(row)); - } else { - if (index.stack().empty()) - return; - auto& undo = index.stack().back(); - if (undo.old_values.empty() && undo.new_ids.empty() && undo.removed_values.empty()) - return; - deltas.push_back({}); - auto& delta = deltas.back(); - delta.name = name; - for (auto& old : undo.old_values) { - auto& row = index.get(old.first); - if (include_delta(old.second, row)) - delta.rows.obj.emplace_back(true, pack_row(row)); - } - for (auto& old : undo.removed_values) - delta.rows.obj.emplace_back(false, pack_row(old.second)); - for (auto id : undo.new_ids) { - auto& row = index.get(id); - delta.rows.obj.emplace_back(true, pack_row(row)); - } - } - }; - - process_table("account", db.get_index(), pack_row); - process_table("account_metadata", db.get_index(), pack_row); - process_table("code", db.get_index(), pack_row); - - process_table("contract_table", db.get_index(), pack_row); - process_table("contract_row", db.get_index(), pack_contract_row); - process_table("contract_index64", db.get_index(), pack_contract_row); - process_table("contract_index128", db.get_index(), pack_contract_row); - process_table("contract_index256", db.get_index(), pack_contract_row); - process_table("contract_index_double", db.get_index(), pack_contract_row); - process_table("contract_index_long_double", db.get_index(), pack_contract_row); - - process_table("global_property", db.get_index(), pack_row); - process_table("generated_transaction", db.get_index(), pack_row); - process_table("protocol_state", db.get_index(), pack_row); - - process_table("permission", db.get_index(), pack_row); - process_table("permission_link", db.get_index(), pack_row); - - process_table("resource_limits", db.get_index(), pack_row); - process_table("resource_usage", db.get_index(), pack_row); - process_table("resource_limits_state", db.get_index(), pack_row); - process_table("resource_limits_config", db.get_index(), pack_row); - - auto deltas_bin = zlib_compress_bytes(fc::raw::pack(deltas)); - EOS_ASSERT(deltas_bin.size() == (uint32_t)deltas_bin.size(), plugin_exception, "deltas is too big"); - state_history_log_header header{.magic = ship_magic(ship_current_version), - .block_id = block_state->block->id(), - .payload_size = sizeof(uint32_t) + deltas_bin.size()}; - chain_state_log->write_entry(header, block_state->block->previous, [&](auto& stream) { - uint32_t s = (uint32_t)deltas_bin.size(); - stream.write((char*)&s, sizeof(s)); - if (!deltas_bin.empty()) - stream.write(deltas_bin.data(), deltas_bin.size()); - }); - } // store_chain_state }; // state_history_plugin_impl state_history_plugin::state_history_plugin() @@ -587,6 +473,22 @@ void state_history_plugin::set_program_options(options_description& cli, options auto options = cfg.add_options(); options("state-history-dir", bpo::value()->default_value("state-history"), "the location of the state-history directory (absolute path or relative to application data dir)"); + options("state-history-retained-dir", bpo::value()->default_value(""), + "the location of the state history retained directory (absolute path or relative to state-history dir).\n" + "If the value is empty, it is set to the value of state-history directory."); + options("state-history-archive-dir", bpo::value()->default_value("archive"), + "the location of the state history archive directory (absolute path or relative to state-history dir).\n" + "If the value is empty, blocks files beyond the retained limit will be deleted.\n" + "All files in the archive directory are completely under user's control, i.e. they won't be accessed by nodeos anymore."); + options("state-history-stride", bpo::value()->default_value(UINT32_MAX), + "split the state history log files when the block number is the multiple of the stride\n" + "When the stride is reached, the current history log and index will be renamed '*-history--.log/index'\n" + "and a new current history log and index will be created with the most recent blocks. All files following\n" + "this format will be used to construct an extended history log."); + options("max-retained-history-files", bpo::value()->default_value(UINT32_MAX), + "the maximum number of history file groups to retain so that the blocks in those files can be queried.\n" + "When the number is reached, the oldest history file would be moved to archive dir or deleted if the archive dir is empty.\n" + "The retained history log files should not be manipulated by users." ); cli.add_options()("delete-state-history", bpo::bool_switch()->default_value(false), "clear state history files"); options("trace-history", bpo::bool_switch()->default_value(false), "enable trace history"); options("chain-state-history", bpo::bool_switch()->default_value(false), "enable chain state history"); @@ -595,6 +497,8 @@ void state_history_plugin::set_program_options(options_description& cli, options "your internal network."); options("trace-history-debug-mode", bpo::bool_switch()->default_value(false), "enable debug mode for trace history"); + options("context-free-data-compression", bpo::value()->default_value("zlib"), + "compression mode for context free data in transaction traces. Supported options are \"zlib\" and \"none\""); } void state_history_plugin::plugin_initialize(const variables_map& options) { @@ -606,7 +510,7 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { EOS_ASSERT(my->chain_plug, chain::missing_chain_plugin_exception, ""); auto& chain = my->chain_plug->chain(); my->applied_transaction_connection.emplace( - chain.applied_transaction.connect([&](std::tuple t) { + chain.applied_transaction.connect([&](std::tuple t) { my->on_applied_transaction(std::get<0>(t), std::get<1>(t)); })); my->accepted_block_connection.emplace( @@ -614,12 +518,21 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { my->block_start_connection.emplace( chain.block_start.connect([&](uint32_t block_num) { my->on_block_start(block_num); })); - auto dir_option = options.at("state-history-dir").as(); - boost::filesystem::path state_history_dir; + auto dir_option = options.at("state-history-dir").as(); + + static eosio::state_history_config config; + if (dir_option.is_relative()) - state_history_dir = app().data_dir() / dir_option; + config.log_dir = app().data_dir() / dir_option; else - state_history_dir = dir_option; + config.log_dir = dir_option; + if (auto resmon_plugin = app().find_plugin()) + resmon_plugin->monitor_directory(config.log_dir); + + config.retained_dir = options.at("state-history-retained-dir").as(); + config.archive_dir = options.at("state-history-archive-dir").as(); + config.stride = options.at("state-history-stride").as(); + config.max_retained_files = options.at("max-retained-history-files").as(); auto ip_port = options.at("state-history-endpoint").as(); auto port = ip_port.substr(ip_port.find(':') + 1, ip_port.size()); @@ -629,26 +542,36 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { idump((ip_port)(host)(port)); if (options.at("delete-state-history").as()) { - ilog("Deleting state history"); - boost::filesystem::remove_all(state_history_dir); + fc_ilog(_log, "Deleting state history"); + boost::filesystem::remove_all(config.log_dir); } - boost::filesystem::create_directories(state_history_dir); - - if (options.at("trace-history-debug-mode").as()) { - my->trace_debug_mode = true; + boost::filesystem::create_directories(config.log_dir); + + if (options.at("trace-history").as()) { + my->trace_log.emplace(config); + if (options.at("trace-history-debug-mode").as()) + my->trace_log->trace_debug_mode = true; + + auto compression = options.at("context-free-data-compression").as(); + if (compression == "zlib") { + my->trace_log->compression = state_history::compression_type::zlib; + } else if (compression == "none") { + my->trace_log->compression = state_history::compression_type::none; + } else { + throw bpo::validation_error(bpo::validation_error::invalid_option_value); + } } - if (options.at("trace-history").as()) - my->trace_log.emplace("trace_history", (state_history_dir / "trace_history.log").string(), - (state_history_dir / "trace_history.index").string()); if (options.at("chain-state-history").as()) - my->chain_state_log.emplace("chain_state_history", (state_history_dir / "chain_state_history.log").string(), - (state_history_dir / "chain_state_history.index").string()); + my->chain_state_log.emplace(config); } FC_LOG_AND_RETHROW() } // state_history_plugin::plugin_initialize -void state_history_plugin::plugin_startup() { my->listen(); } +void state_history_plugin::plugin_startup() { + handle_sighup(); // setup logging + my->listen(); +} void state_history_plugin::plugin_shutdown() { my->applied_transaction_connection.reset(); @@ -659,4 +582,8 @@ void state_history_plugin::plugin_shutdown() { my->stopping = true; } +void state_history_plugin::handle_sighup() { + fc::logger::update( logger_name, _log ); +} + } // namespace eosio diff --git a/plugins/test_control_api_plugin/test_control.swagger.yaml b/plugins/test_control_api_plugin/test_control.swagger.yaml deleted file mode 100644 index 906fe76ea8c..00000000000 --- a/plugins/test_control_api_plugin/test_control.swagger.yaml +++ /dev/null @@ -1,58 +0,0 @@ -openapi: 3.0.0 -info: - title: Test Control API - version: 1.0.0 - license: - name: MIT - url: https://opensource.org/licenses/MIT - contact: - url: https://eos.io -tags: - - name: eosio -servers: - - url: '{protocol}://{host}:{port}/v1/' - variables: - protocol: - enum: - - http - - https - default: http - host: - default: localhost - port: - default: "8080" -components: - schemas: {} -paths: - /test_control/kill_node_or_producer: - post: - tags: - - TestControl - summary: kill_node_or_producer - description: Kills node or producer - operationId: kill_node_or_producer - parameters: [] - requestBody: - content: - application/json: - schema: - type: object - required: - - params - properties: - params: - type: object - properties: - producer: - $ref: 'https://eosio.github.io/schemata/v2.0/oas/Name.yaml' - where_in_sequence: - type: integer - based_on_lib: - type: integer - responses: - '200': - description: OK - content: - application/json: - schema: - description: Returns Nothing diff --git a/plugins/test_control_api_plugin/test_control_api_plugin.cpp b/plugins/test_control_api_plugin/test_control_api_plugin.cpp index 0bac41fa1c0..7aefdaf464b 100644 --- a/plugins/test_control_api_plugin/test_control_api_plugin.cpp +++ b/plugins/test_control_api_plugin/test_control_api_plugin.cpp @@ -31,26 +31,26 @@ struct async_result_visitor : public fc::visitor { } }; -#define CALL(api_name, api_handle, api_namespace, call_name, http_response_code) \ +#define CALL_WITH_API_400(api_name, api_handle, api_namespace, call_name, http_response_code, params_type) \ {std::string("/v1/" #api_name "/" #call_name), \ [api_handle](string, string body, url_response_callback cb) mutable { \ try { \ - if (body.empty()) body = "{}"; \ - fc::variant result( api_handle.call_name(fc::json::from_string(body).as()) ); \ + auto params = parse_params(body);\ + fc::variant result( api_handle.call_name( std::move(params) ) ); \ cb(http_response_code, std::move(result)); \ } catch (...) { \ http_plugin::handle_exception(#api_name, #call_name, body, cb); \ } \ }} -#define TEST_CONTROL_RW_CALL(call_name, http_response_code) CALL(test_control, rw_api, test_control_apis::read_write, call_name, http_response_code) +#define TEST_CONTROL_RW_CALL(call_name, http_response_code, params_type) CALL_WITH_API_400(test_control, rw_api, test_control_apis::read_write, call_name, http_response_code, params_type) void test_control_api_plugin::plugin_startup() { my.reset(new test_control_api_plugin_impl(app().get_plugin().chain())); auto rw_api = app().get_plugin().get_read_write_api(); app().get_plugin().add_api({ - TEST_CONTROL_RW_CALL(kill_node_on_producer, 202) + TEST_CONTROL_RW_CALL(kill_node_on_producer, 202, http_params_types::params_required) }); } diff --git a/plugins/test_control_plugin/test_control_plugin.cpp b/plugins/test_control_plugin/test_control_plugin.cpp index 3ec82e722b9..b922971380a 100644 --- a/plugins/test_control_plugin/test_control_plugin.cpp +++ b/plugins/test_control_plugin/test_control_plugin.cpp @@ -1,5 +1,4 @@ #include -#include #include namespace fc { class variant; } @@ -21,8 +20,8 @@ class test_control_plugin_impl { void applied_irreversible_block(const chain::block_state_ptr& bsp); void process_next_block_state(const chain::block_state_ptr& bsp); - fc::optional _accepted_block_connection; - fc::optional _irreversible_block_connection; + std::optional _accepted_block_connection; + std::optional _irreversible_block_connection; chain::controller& _chain; account_name _producer; int32_t _where_in_sequence; diff --git a/plugins/trace_api_plugin/abi_data_handler.cpp b/plugins/trace_api_plugin/abi_data_handler.cpp index 095ab2cdcb8..06462d7e5a4 100644 --- a/plugins/trace_api_plugin/abi_data_handler.cpp +++ b/plugins/trace_api_plugin/abi_data_handler.cpp @@ -9,10 +9,13 @@ namespace eosio::trace_api { std::make_shared(abi, chain::abi_serializer::create_yield_function(fc::microseconds::maximum()))); } - fc::variant abi_data_handler::process_data(const action_trace_v0& action, const yield_function& yield ) { - if (abi_serializer_by_account.count(action.account) > 0) { - const auto& serializer_p = abi_serializer_by_account.at(action.account); - auto type_name = serializer_p->get_action_type(action.action); + std::tuple> abi_data_handler::serialize_to_variant(const std::variant & action, const yield_function& yield ) { + auto account = std::visit([](auto &&action) -> auto { return action.account; }, action); + + if (abi_serializer_by_account.count(account) > 0) { + const auto &serializer_p = abi_serializer_by_account.at(account); + auto action_name = std::visit([](auto &&action) -> auto { return action.action; }, action); + auto type_name = serializer_p->get_action_type(action_name); if (!type_name.empty()) { try { @@ -22,7 +25,15 @@ namespace eosio::trace_api { EOS_ASSERT( recursion_depth < chain::abi_serializer::max_recursion_depth, chain::abi_recursion_depth_exception, "exceeded max_recursion_depth ${r} ", ("r", chain::abi_serializer::max_recursion_depth) ); }; - return serializer_p->binary_to_variant(type_name, action.data, abi_yield); + return std::visit([&](auto &&action) -> std::tuple> { + using T = std::decay_t; + if constexpr (std::is_same_v) { + return {serializer_p->binary_to_variant(type_name, action.data, abi_yield), {}}; + } else { + return {serializer_p->binary_to_variant(type_name, action.data, abi_yield), + {serializer_p->binary_to_variant(type_name, action.return_value, abi_yield)}}; + } + }, action); } catch (...) { except_handler(MAKE_EXCEPTION_WITH_CONTEXT(std::current_exception())); } diff --git a/plugins/trace_api_plugin/compressed_file.cpp b/plugins/trace_api_plugin/compressed_file.cpp index fc9aee8e0d2..bbd3ee6fd2c 100644 --- a/plugins/trace_api_plugin/compressed_file.cpp +++ b/plugins/trace_api_plugin/compressed_file.cpp @@ -135,7 +135,7 @@ struct compressed_file_impl { }); // special case when there is a seek point that is exact - if ( iter != seek_point_map.end() && std::get<0>(*iter) == loc ) { + if ( iter != seek_point_map.end() && std::get<0>(*iter) == static_cast(loc) ) { file.seek(std::get<1>(*iter)); return; } @@ -233,7 +233,7 @@ bool compressed_file::process( const fc::path& input_path, const fc::path& outpu auto output_buffer = std::vector(buffer_size); auto bytes_remaining_before_sync = seek_point_stride; - int next_sync_point = 0; + unsigned int next_sync_point = 0; // process a single chunk of input completely, // this may sometime loop multiple times if the compressor state combined with input data creates more than a diff --git a/plugins/trace_api_plugin/include/eosio/trace_api/abi_data_handler.hpp b/plugins/trace_api_plugin/include/eosio/trace_api/abi_data_handler.hpp index 2277ef5f244..f28d7c33f31 100644 --- a/plugins/trace_api_plugin/include/eosio/trace_api/abi_data_handler.hpp +++ b/plugins/trace_api_plugin/include/eosio/trace_api/abi_data_handler.hpp @@ -31,13 +31,13 @@ namespace eosio { void add_abi( const chain::name& name, const chain::abi_def& abi ); /** - * Given an action trace, produce a variant that represents the `data` field in the trace + * Given an action trace, produce a tuple representing the `data` and `return_value` fields in the trace * * @param action - trace of the action including metadata necessary for finding the ABI * @param yield - a yield function to allow cooperation during long running tasks - * @return variant representing the `data` field of the action interpreted by known ABIs OR an empty variant + * @return tuple with the first element is a variant representing the `data` field of the action interpreted by known ABIs OR an empty variant, the second element representing the `return_value` field of the trace. */ - fc::variant process_data( const action_trace_v0& action, const yield_function& yield ); + std::tuple> serialize_to_variant(const std::variant & action, const yield_function& yield ); /** * Utility class that allows mulitple request_handlers to share the same abi_data_handler @@ -48,8 +48,8 @@ namespace eosio { :handler(handler) {} - fc::variant process_data( const action_trace_v0& action, const yield_function& yield ) { - return handler->process_data(action, yield); + std::tuple> serialize_to_variant( const std::variant & action, const yield_function& yield ) { + return handler->serialize_to_variant(action, yield); } std::shared_ptr handler; diff --git a/plugins/trace_api_plugin/include/eosio/trace_api/chain_extraction.hpp b/plugins/trace_api_plugin/include/eosio/trace_api/chain_extraction.hpp index 16404c1dd4e..e79f38da4b6 100644 --- a/plugins/trace_api_plugin/include/eosio/trace_api/chain_extraction.hpp +++ b/plugins/trace_api_plugin/include/eosio/trace_api/chain_extraction.hpp @@ -26,8 +26,8 @@ class chain_extraction_impl_type { {} /// connect to chain controller applied_transaction signal - void signal_applied_transaction( const chain::transaction_trace_ptr& trace, const chain::signed_transaction& strx ) { - on_applied_transaction( trace, strx ); + void signal_applied_transaction( const chain::transaction_trace_ptr& trace, const chain::packed_transaction_ptr& ptrx ) { + on_applied_transaction( trace, ptrx ); } /// connect to chain controller accepted_block signal @@ -46,8 +46,7 @@ class chain_extraction_impl_type { } private: - - void on_applied_transaction(const chain::transaction_trace_ptr& trace, const chain::signed_transaction& t) { + void on_applied_transaction(const chain::transaction_trace_ptr& trace, const chain::packed_transaction_ptr& t) { if( !trace->receipt ) return; // include only executed transactions; soft_fail included so that onerror (and any inlines via onerror) are included if((trace->receipt->status != chain::transaction_receipt_header::executed && @@ -55,11 +54,11 @@ class chain_extraction_impl_type { return; } if( chain::is_onblock( *trace )) { - onblock_trace.emplace( cache_trace{trace, static_cast(t), t.signatures} ); + onblock_trace.emplace( cache_trace{trace, t} ); } else if( trace->failed_dtrx_trace ) { - cached_traces[trace->failed_dtrx_trace->id] = {trace, static_cast(t), t.signatures}; + cached_traces[trace->failed_dtrx_trace->id] = {trace, t}; } else { - cached_traces[trace->id] = {trace, static_cast(t), t.signatures}; + cached_traces[trace->id] = {trace, t}; } } @@ -82,22 +81,24 @@ class chain_extraction_impl_type { void store_block_trace( const chain::block_state_ptr& block_state ) { try { - block_trace_v1 bt = create_block_trace_v1( block_state ); + using transaction_trace_t = transaction_trace_v2; + + auto bt = create_block_trace( block_state ); - std::vector& traces = bt.transactions_v1; + std::vector& traces = std::get>(bt.transactions); traces.reserve( block_state->block->transactions.size() + 1 ); if( onblock_trace ) - traces.emplace_back( to_transaction_trace_v1( *onblock_trace )); + traces.emplace_back( to_transaction_trace( *onblock_trace )); for( const auto& r : block_state->block->transactions ) { transaction_id_type id; - if( r.trx.contains()) { - id = r.trx.get(); + if( std::holds_alternative(r.trx)) { + id = std::get(r.trx); } else { - id = r.trx.get().id(); + id = std::get(r.trx).id(); } const auto it = cached_traces.find( id ); if( it != cached_traces.end() ) { - traces.emplace_back( to_transaction_trace_v1( it->second )); + traces.emplace_back( to_transaction_trace( it->second )); } } clear_caches(); @@ -121,8 +122,8 @@ class chain_extraction_impl_type { StoreProvider store; exception_handler except_handler; std::map cached_traces; - fc::optional onblock_trace; + std::optional onblock_trace; }; -}} \ No newline at end of file +}} diff --git a/plugins/trace_api_plugin/include/eosio/trace_api/data_log.hpp b/plugins/trace_api_plugin/include/eosio/trace_api/data_log.hpp index 9f1af1f3922..e4906055a30 100644 --- a/plugins/trace_api_plugin/include/eosio/trace_api/data_log.hpp +++ b/plugins/trace_api_plugin/include/eosio/trace_api/data_log.hpp @@ -6,9 +6,10 @@ namespace eosio { namespace trace_api { - using data_log_entry = fc::static_variant< + using data_log_entry = std::variant< block_trace_v0, - block_trace_v1 + block_trace_v1, + block_trace_v2 >; }} diff --git a/plugins/trace_api_plugin/include/eosio/trace_api/extract_util.hpp b/plugins/trace_api_plugin/include/eosio/trace_api/extract_util.hpp index 0e98af8b87a..4bbef986e1e 100644 --- a/plugins/trace_api_plugin/include/eosio/trace_api/extract_util.hpp +++ b/plugins/trace_api_plugin/include/eosio/trace_api/extract_util.hpp @@ -5,13 +5,17 @@ namespace eosio { namespace trace_api { -/// Used by to_transaction_trace_v0 for creation of action_trace_v0 -inline action_trace_v0 to_action_trace_v0( const chain::action_trace& at ) { - action_trace_v0 r; +/// Used by to_transaction_trace for creation of action_trace_v0 or action_trace_v1 +template +inline ActionTrace to_action_trace( const chain::action_trace& at ) { + ActionTrace r; r.receiver = at.receiver; r.account = at.act.account; r.action = at.act.name; r.data = at.act.data; + if constexpr(std::is_same_v){ + r.return_value = at.return_value; + } if( at.receipt ) { r.global_sequence = at.receipt->global_sequence; } @@ -22,69 +26,50 @@ inline action_trace_v0 to_action_trace_v0( const chain::action_trace& at ) { return r; } -/// @return transaction_trace_v0 with populated action_trace_v0 -inline transaction_trace_v0 to_transaction_trace_v0( const chain::transaction_trace_ptr& t ) { - transaction_trace_v0 r; - if( !t->failed_dtrx_trace ) { - r.id = t->id; +template +inline TransactionTrace to_transaction_trace( const cache_trace& t ) { + TransactionTrace r; + if( !t.trace->failed_dtrx_trace ) { + r.id = t.trace->id; } else { - r.id = t->failed_dtrx_trace->id; // report the failed trx id since that is the id known to user + r.id = t.trace->failed_dtrx_trace->id; // report the failed trx id since that is the id known to user + } + + if constexpr(std::is_same_v || std::is_same_v){ + if (t.trace->receipt) { + r.status = t.trace->receipt->status; + r.cpu_usage_us = t.trace->receipt->cpu_usage_us; + r.net_usage_words = t.trace->receipt->net_usage_words; + } + auto sigs = t.trx->get_signatures(); + if( sigs ) r.signatures = *sigs; + r.trx_header = static_cast( t.trx->get_transaction() ); } - r.actions.reserve( t->action_traces.size()); - for( const auto& at : t->action_traces ) { + + using action_trace_t = std::conditional_t, action_trace_v1, action_trace_v0>; + + r.actions = std::vector(); + std::get>(r.actions).reserve( t.trace->action_traces.size()); + for( const auto& at : t.trace->action_traces ) { if( !at.context_free ) { // not including CFA at this time - r.actions.emplace_back( to_action_trace_v0( at )); + std::get>(r.actions).emplace_back( to_action_trace(at) ); } } - return r; -} -inline transaction_trace_v1 to_transaction_trace_v1( const cache_trace& t ) { - transaction_trace_v1 r; - if( !t.trace->failed_dtrx_trace ) { - r.id = t.trace->id; - } else { - r.id = t.trace->failed_dtrx_trace->id; // report the failed trx id since that is the id known to user - } - if (t.trace->receipt) { - r.status = t.trace->receipt->status; - r.cpu_usage_us = t.trace->receipt->cpu_usage_us; - r.net_usage_words = t.trace->receipt->net_usage_words; - } - r.signatures = t.trx_signatures; - r.trx_header = t.trx_header; - r.actions.reserve( t.trace->action_traces.size()); - for( const auto& at : t.trace->action_traces ) { - if( !at.context_free ) { // not including CFA at this time - r.actions.emplace_back( to_action_trace_v0( at )); - } - } - return r; + return r; } -/// @return block_trace_v0 without any transaction_trace_v0 -inline block_trace_v0 create_block_trace_v0( const chain::block_state_ptr& bsp ) { - block_trace_v0 r; +inline block_trace_v2 create_block_trace( const chain::block_state_ptr& bsp ) { + block_trace_v2 r; r.id = bsp->id; r.number = bsp->block_num; r.previous_id = bsp->block->previous; r.timestamp = bsp->block->timestamp; r.producer = bsp->block->producer; + r.schedule_version = bsp->block->schedule_version; + r.transaction_mroot = bsp->block->transaction_mroot; + r.action_mroot = bsp->block->action_mroot; return r; } -/// @return block_trace_v1 without any transaction_trace_v1 -inline block_trace_v1 create_block_trace_v1( const chain::block_state_ptr& bsp ) { - block_trace_v1 r; - r.id = bsp->id; - r.number = bsp->block_num; - r.previous_id = bsp->block->previous; - r.timestamp = bsp->block->timestamp; - r.producer = bsp->block->producer; - r.schedule_version = bsp->block->schedule_version; - r.transaction_mroot = bsp->block->transaction_mroot; - r.action_mroot = bsp->block->action_mroot; - return r; -} - } } diff --git a/plugins/trace_api_plugin/include/eosio/trace_api/metadata_log.hpp b/plugins/trace_api_plugin/include/eosio/trace_api/metadata_log.hpp index 7bcc73d3ed2..52e6eb94fd3 100644 --- a/plugins/trace_api_plugin/include/eosio/trace_api/metadata_log.hpp +++ b/plugins/trace_api_plugin/include/eosio/trace_api/metadata_log.hpp @@ -15,7 +15,7 @@ namespace eosio { namespace trace_api { uint32_t lib; }; - using metadata_log_entry = fc::static_variant< + using metadata_log_entry = std::variant< block_entry_v0, lib_entry_v0 >; diff --git a/plugins/trace_api_plugin/include/eosio/trace_api/request_handler.hpp b/plugins/trace_api_plugin/include/eosio/trace_api/request_handler.hpp index 3e66b2c82bf..be19aa69c87 100644 --- a/plugins/trace_api_plugin/include/eosio/trace_api/request_handler.hpp +++ b/plugins/trace_api_plugin/include/eosio/trace_api/request_handler.hpp @@ -6,7 +6,7 @@ #include namespace eosio::trace_api { - using data_handler_function = std::function; + using data_handler_function = std::function>( const std::variant & action_trace_t, const yield_function&)>; namespace detail { class response_formatter { @@ -43,8 +43,10 @@ namespace eosio::trace_api { yield(); - auto data_handler = [this](const action_trace_v0& action, const yield_function& yield) -> fc::variant { - return data_handler_provider.process_data(action, yield); + auto data_handler = [this](const auto& action, const yield_function& yield) -> std::tuple> { + return std::visit([&](const auto& action_trace_t) { + return data_handler_provider.serialize_to_variant(action_trace_t, yield); + }, action); }; return detail::response_formatter::process_block(std::get<0>(*data), std::get<1>(*data), data_handler, yield); diff --git a/plugins/trace_api_plugin/include/eosio/trace_api/store_provider.hpp b/plugins/trace_api_plugin/include/eosio/trace_api/store_provider.hpp index 60a8990cd18..7993c468f8d 100644 --- a/plugins/trace_api_plugin/include/eosio/trace_api/store_provider.hpp +++ b/plugins/trace_api_plugin/include/eosio/trace_api/store_provider.hpp @@ -246,7 +246,8 @@ namespace eosio::trace_api { store_provider(const boost::filesystem::path& slice_dir, uint32_t stride_width, std::optional minimum_irreversible_history_blocks, std::optional minimum_uncompressed_irreversible_history_blocks, size_t compression_seek_point_stride); - void append(const block_trace_v1& bt); + template + void append(const BlockTrace& bt); void append_lib(uint32_t lib); /** diff --git a/plugins/trace_api_plugin/include/eosio/trace_api/trace.hpp b/plugins/trace_api_plugin/include/eosio/trace_api/trace.hpp index 3e0937fa626..68bc1858db4 100644 --- a/plugins/trace_api_plugin/include/eosio/trace_api/trace.hpp +++ b/plugins/trace_api_plugin/include/eosio/trace_api/trace.hpp @@ -21,6 +21,10 @@ namespace eosio { namespace trace_api { chain::bytes data = {}; }; + struct action_trace_v1 : public action_trace_v0 { + chain::bytes return_value = {}; + }; + struct transaction_trace_v0 { using status_type = chain::transaction_receipt_header::status_enum; @@ -36,6 +40,18 @@ namespace eosio { namespace trace_api { chain::transaction_header trx_header = {}; }; + struct transaction_trace_v2 { + using status_type = chain::transaction_receipt_header::status_enum; + + chain::transaction_id_type id = {}; + std::variant> actions = {}; + fc::enum_type status = {}; + uint32_t cpu_usage_us = 0; + fc::unsigned_int net_usage_words; + std::vector signatures = {}; + chain::transaction_header trx_header = {}; + }; + struct block_trace_v0 { chain::block_id_type id = {}; uint32_t number = {}; @@ -52,18 +68,31 @@ namespace eosio { namespace trace_api { std::vector transactions_v1 = {}; }; + struct block_trace_v2 { + chain::block_id_type id = {}; + uint32_t number = {}; + chain::block_id_type previous_id = {}; + chain::block_timestamp_type timestamp = chain::block_timestamp_type(0); + chain::name producer = {}; + chain::checksum256_type transaction_mroot = {}; + chain::checksum256_type action_mroot = {}; + uint32_t schedule_version = {}; + std::variant> transactions = {}; + }; + struct cache_trace { chain::transaction_trace_ptr trace; - chain::transaction_header trx_header; - std::vector trx_signatures; + chain::packed_transaction_ptr trx; }; } } FC_REFLECT(eosio::trace_api::authorization_trace_v0, (account)(permission)) FC_REFLECT(eosio::trace_api::action_trace_v0, (global_sequence)(receiver)(account)(action)(authorization)(data)) +FC_REFLECT_DERIVED(eosio::trace_api::action_trace_v1, (eosio::trace_api::action_trace_v0),(return_value)) FC_REFLECT(eosio::trace_api::transaction_trace_v0, (id)(actions)) FC_REFLECT_DERIVED(eosio::trace_api::transaction_trace_v1, (eosio::trace_api::transaction_trace_v0), (status)(cpu_usage_us)(net_usage_words)(signatures)(trx_header)) +FC_REFLECT(eosio::trace_api::transaction_trace_v2, (id)(actions)(status)(cpu_usage_us)(net_usage_words)(signatures)(trx_header)) FC_REFLECT(eosio::trace_api::block_trace_v0, (id)(number)(previous_id)(timestamp)(producer)(transactions)) FC_REFLECT_DERIVED(eosio::trace_api::block_trace_v1, (eosio::trace_api::block_trace_v0), (transaction_mroot)(action_mroot)(schedule_version)(transactions_v1)) -FC_REFLECT(eosio::trace_api::cache_trace, (trace)(trx_header)(trx_signatures)) \ No newline at end of file +FC_REFLECT(eosio::trace_api::block_trace_v2, (id)(number)(previous_id)(timestamp)(producer)(transaction_mroot)(action_mroot)(schedule_version)(transactions)) diff --git a/plugins/trace_api_plugin/request_handler.cpp b/plugins/trace_api_plugin/request_handler.cpp index 42560ba3e85..03bcf835d21 100644 --- a/plugins/trace_api_plugin/request_handler.cpp +++ b/plugins/trace_api_plugin/request_handler.cpp @@ -27,7 +27,8 @@ namespace { } - fc::variants process_actions(const std::vector& actions, const data_handler_function& data_handler, const yield_function& yield ) { + template + fc::variants process_actions(const std::vector& actions, const data_handler_function & data_handler, const yield_function& yield ) { fc::variants result; result.reserve(actions.size()); @@ -42,89 +43,116 @@ namespace { yield(); const auto& a = actions.at(index); - auto action_variant = fc::mutable_variant_object() - ("global_sequence", a.global_sequence) + auto common_mvo = fc::mutable_variant_object(); + + common_mvo("global_sequence", a.global_sequence) ("receiver", a.receiver.to_string()) ("account", a.account.to_string()) ("action", a.action.to_string()) ("authorization", process_authorizations(a.authorization, yield)) ("data", fc::to_hex(a.data.data(), a.data.size())); - auto params = data_handler(a, yield); - if (!params.is_null()) { - action_variant("params", params); + auto action_variant = fc::mutable_variant_object(); + if constexpr(std::is_same_v){ + action_variant(std::move(common_mvo)); + auto [params, return_data] = data_handler(a, yield); + if (!params.is_null()) { + action_variant("params", params); + } + } + else if constexpr(std::is_same_v){ + action_variant(std::move(common_mvo)); + action_variant("return_value", fc::to_hex(a.return_value.data(),a.return_value.size())) ; + + auto [params, return_data] = data_handler(a, yield); + if (!params.is_null()) { + action_variant("params", params); + } + if(return_data.has_value()){ + action_variant("return_data", *return_data); + } } result.emplace_back( std::move(action_variant) ); } return result; - } - fc::variants process_transactions(const std::vector& transactions, const data_handler_function& data_handler, const yield_function& yield ) { + template + fc::variants process_transactions(const std::vector& transactions, const data_handler_function & data_handler, const yield_function& yield ) { fc::variants result; result.reserve(transactions.size()); for ( const auto& t: transactions) { yield(); + if constexpr(std::is_same_v){ + result.emplace_back( + fc::mutable_variant_object() + ("id", t.id.str()) + ("actions", process_actions(t.actions, data_handler, yield))); + } else { + auto common_mvo = fc::mutable_variant_object(); + common_mvo("status", t.status) + ("cpu_usage_us", t.cpu_usage_us) + ("net_usage_words", t.net_usage_words) + ("signatures", t.signatures) + ("transaction_header", t.trx_header); + + if constexpr(std::is_same_v){ + result.emplace_back( + fc::mutable_variant_object() + ("id", t.id.str()) + ("actions", process_actions(t.actions, data_handler, yield)) + (std::move(common_mvo))); + } + else if constexpr(std::is_same_v){ + result.emplace_back( + fc::mutable_variant_object() + ("id", t.id.str()) + ("actions", process_actions(std::get>(t.actions), data_handler, yield)) + (std::move(common_mvo))); + } + } - result.emplace_back(fc::mutable_variant_object() - ("id", t.id.str()) - ("actions", process_actions(t.actions, data_handler, yield)) - ); } - return result; } - - fc::variants process_transactions(const std::vector& transactions, const data_handler_function& data_handler, const yield_function& yield ) { - fc::variants result; - result.reserve(transactions.size()); - for ( const auto& t: transactions) { - yield(); - result.emplace_back(fc::mutable_variant_object() - ("id", t.id.str()) - ("actions", process_actions(t.actions, data_handler, yield)) - ("status", t.status) - ("cpu_usage_us", t.cpu_usage_us) - ("net_usage_words", t.net_usage_words) - ("signatures", t.signatures) - ("transaction_header", t.trx_header) - ); - } - - return result; - } - } namespace eosio::trace_api::detail { - fc::variant process_block_trace( const block_trace_v0& trace, bool irreversible, const data_handler_function& data_handler, const yield_function& yield ) { - return fc::mutable_variant_object() - ("id", trace.id.str() ) - ("number", trace.number ) - ("previous_id", trace.previous_id.str() ) - ("status", irreversible ? "irreversible" : "pending" ) - ("timestamp", to_iso8601_datetime(trace.timestamp)) - ("producer", trace.producer.to_string()) - ("transactions", process_transactions(trace.transactions, data_handler, yield )); - } - - fc::variant process_block_trace( const block_trace_v1& trace, bool irreversible, const data_handler_function& data_handler, const yield_function& yield ) { - return fc::mutable_variant_object() - ("id", trace.id.str() ) - ("number", trace.number ) - ("previous_id", trace.previous_id.str() ) - ("status", irreversible ? "irreversible" : "pending" ) - ("timestamp", to_iso8601_datetime(trace.timestamp)) - ("producer", trace.producer.to_string()) - ("transaction_mroot", trace.transaction_mroot) - ("action_mroot", trace.action_mroot) - ("schedule_version", trace.schedule_version) - ("transactions", process_transactions(trace.transactions_v1, data_handler, yield )); - } fc::variant response_formatter::process_block( const data_log_entry& trace, bool irreversible, const data_handler_function& data_handler, const yield_function& yield ) { - if (trace.contains()) return process_block_trace(trace.get(), irreversible, data_handler, yield); - else return process_block_trace(trace.get(), irreversible, data_handler, yield); + auto common_mvo = std::visit([&](auto&& arg) -> fc::mutable_variant_object { + return fc::mutable_variant_object() + ("id", arg.id.str()) + ("number", arg.number ) + ("previous_id", arg.previous_id.str()) + ("status", irreversible ? "irreversible" : "pending" ) + ("timestamp", to_iso8601_datetime(arg.timestamp)) + ("producer", arg.producer.to_string());}, trace); + + if (std::holds_alternative (trace)){ + auto& block_trace = std::get(trace); + return fc::mutable_variant_object() + (std::move(common_mvo)) + ("transactions", process_transactions(block_trace.transactions, data_handler, yield )); + }else if(std::holds_alternative(trace)){ + auto& block_trace = std::get(trace); + return fc::mutable_variant_object() + (std::move(common_mvo)) + ("transaction_mroot", block_trace.transaction_mroot) + ("action_mroot", block_trace.action_mroot) + ("schedule_version", block_trace.schedule_version) + ("transactions", process_transactions( block_trace.transactions_v1, data_handler, yield )) ; + }else if(std::holds_alternative(trace)){ + auto& block_trace = std::get(trace); + return fc::mutable_variant_object() + (std::move(common_mvo)) + ("transaction_mroot", block_trace.transaction_mroot) + ("action_mroot", block_trace.action_mroot) + ("schedule_version", block_trace.schedule_version) + ("transactions", process_transactions( std::get>(block_trace.transactions), data_handler, yield )) ; + }else{ + return fc::mutable_variant_object(); + } } } diff --git a/plugins/trace_api_plugin/store_provider.cpp b/plugins/trace_api_plugin/store_provider.cpp index a540db24dd7..92f8811b508 100644 --- a/plugins/trace_api_plugin/store_provider.cpp +++ b/plugins/trace_api_plugin/store_provider.cpp @@ -14,7 +14,7 @@ namespace { std::string make_filename(const char* slice_prefix, const char* slice_ext, uint32_t slice_number, uint32_t slice_width) { char filename[_max_filename_size] = {}; const uint32_t slice_start = slice_number * slice_width; - const int size_written = snprintf(filename, _max_filename_size, "%s%010d-%010d%s", slice_prefix, slice_start, (slice_start + slice_width), slice_ext); + const unsigned int size_written = snprintf(filename, _max_filename_size, "%s%010d-%010d%s", slice_prefix, slice_start, (slice_start + slice_width), slice_ext); // assert that _max_filename_size is correct if ( size_written >= _max_filename_size ) { const std::string max_size_str = std::to_string(_max_filename_size - 1); // dropping null character from size @@ -34,7 +34,8 @@ namespace eosio::trace_api { : _slice_directory(slice_dir, stride_width, minimum_irreversible_history_blocks, minimum_uncompressed_irreversible_history_blocks, compression_seek_point_stride) { } - void store_provider::append(const block_trace_v1& bt) { + template + void store_provider::append(const BlockTrace& bt) { fc::cfile trace; fc::cfile index; const uint32_t slice_number = _slice_directory.slice_number(bt.number); @@ -46,6 +47,9 @@ namespace eosio::trace_api { append_store(be, index); } + template void store_provider::append(const block_trace_v1& bt); + template void store_provider::append(const block_trace_v2& bt); + void store_provider::append_lib(uint32_t lib) { fc::cfile index; const uint32_t slice_number = _slice_directory.slice_number(lib); @@ -59,13 +63,13 @@ namespace eosio::trace_api { std::optional trace_offset; bool irreversible = false; uint64_t offset = scan_metadata_log_from(block_height, 0, [&block_height, &trace_offset, &irreversible](const metadata_log_entry& e) -> bool { - if (e.contains()) { - const auto& block = e.get(); + if (std::holds_alternative(e)) { + const auto& block = std::get(e); if (block.number == block_height) { trace_offset = block.offset; } - } else if (e.contains()) { - auto lib = e.get().lib; + } else if (std::holds_alternative(e)) { + auto lib = std::get(e).lib; if (lib >= block_height) { irreversible = true; return false; diff --git a/plugins/trace_api_plugin/test/include/eosio/trace_api/test_common.hpp b/plugins/trace_api_plugin/test/include/eosio/trace_api/test_common.hpp index 0369a41aec6..b9ee89827c3 100644 --- a/plugins/trace_api_plugin/test/include/eosio/trace_api/test_common.hpp +++ b/plugins/trace_api_plugin/test/include/eosio/trace_api/test_common.hpp @@ -1,13 +1,10 @@ #pragma once -#include #include #include #include #include -#include -#include #include #include @@ -31,81 +28,18 @@ namespace eosio::trace_api { return chain::asset::from_string(input); } - auto get_private_key( chain::name keyname, std::string role = "owner" ) { - auto secret = fc::sha256::hash( keyname.to_string() + role ); - return chain::private_key_type::regenerate( secret ); - } - - auto get_public_key( chain::name keyname, std::string role = "owner" ) { - return get_private_key( keyname, role ).get_public_key(); - } - chain::bytes make_transfer_data( chain::name from, chain::name to, chain::asset quantity, std::string&& memo) { fc::datastream ps; - fc::raw::pack(ps, from, to, quantity, memo); - chain::bytes result( ps.tellp()); + fc::raw::pack( ps, from, to, quantity, memo ); + chain::bytes result( ps.tellp() ); - if( result.size()) { - fc::datastream ds( result.data(), size_t( result.size())); - fc::raw::pack(ds, from, to, quantity, memo); + if( result.size() ) { + fc::datastream ds( result.data(), size_t( result.size() ) ); + fc::raw::pack( ds, from, to, quantity, memo ); } return result; } - auto make_block_state( chain::block_id_type previous, uint32_t height, uint32_t slot, chain::name producer, - std::vector trxs ) { - chain::signed_block_ptr block = std::make_shared(); - for( auto& trx : trxs ) { - block->transactions.emplace_back( trx ); - } - block->producer = producer; - block->timestamp = chain::block_timestamp_type(slot); - // make sure previous contains correct block # so block_header::block_num() returns correct value - if( previous == chain::block_id_type() ) { - previous._hash[0] &= 0xffffffff00000000; - previous._hash[0] += fc::endian_reverse_u32(height - 1); - } - block->previous = previous; - - auto priv_key = get_private_key( block->producer, "active" ); - auto pub_key = get_public_key( block->producer, "active" ); - - auto prev = std::make_shared(); - auto header_bmroot = chain::digest_type::hash( std::make_pair( block->digest(), prev->blockroot_merkle.get_root())); - auto sig_digest = chain::digest_type::hash( std::make_pair( header_bmroot, prev->pending_schedule.schedule_hash )); - block->producer_signature = priv_key.sign( sig_digest ); - - std::vector signing_keys; - signing_keys.emplace_back( std::move( priv_key )); - auto signer = [&]( chain::digest_type d ) { - std::vector result; - result.reserve( signing_keys.size()); - for( const auto& k: signing_keys ) - result.emplace_back( k.sign( d )); - return result; - }; - chain::pending_block_header_state pbhs; - pbhs.producer = block->producer; - pbhs.timestamp = block->timestamp; - chain::producer_authority_schedule schedule = {0, {chain::producer_authority{block->producer, - chain::block_signing_authority_v0{1, {{pub_key, 1}}}}}}; - pbhs.active_schedule = schedule; - pbhs.valid_block_signing_authority = chain::block_signing_authority_v0{1, {{pub_key, 1}}}; - auto bsp = std::make_shared( - std::move( pbhs ), - std::move( block ), - std::vector(), - chain::protocol_feature_set(), - []( chain::block_timestamp_type timestamp, - const fc::flat_set& cur_features, - const std::vector& new_features ) {}, - signer - ); - bsp->block_num = height; - - return bsp; - } - void to_kv_helper(const fc::variant& v, std::function&& append){ if (v.is_object() ) { const auto& obj = v.get_object(); @@ -164,6 +98,22 @@ namespace eosio::trace_api { lhs.actions == rhs.actions; } + bool operator==(const transaction_trace_v2& lhs, const transaction_trace_v2& rhs) { + return + lhs.id == rhs.id && + lhs.actions == rhs.actions && + lhs.status == rhs.status && + lhs.cpu_usage_us == rhs.cpu_usage_us && + lhs.net_usage_words == rhs.net_usage_words && + lhs.signatures == rhs.signatures && + lhs.trx_header.expiration == rhs.trx_header.expiration && + lhs.trx_header.ref_block_num == rhs.trx_header.ref_block_num && + lhs.trx_header.ref_block_prefix == rhs.trx_header.ref_block_prefix && + lhs.trx_header.max_net_usage_words == rhs.trx_header.max_net_usage_words && + lhs.trx_header.max_cpu_usage_ms == rhs.trx_header.max_cpu_usage_ms && + lhs.trx_header.delay_sec == rhs.trx_header.delay_sec ; + } + bool operator==(const block_trace_v0 &lhs, const block_trace_v0 &rhs) { return lhs.id == rhs.id && @@ -174,11 +124,29 @@ namespace eosio::trace_api { lhs.transactions == rhs.transactions; } + bool operator==(const block_trace_v2 &lhs, const block_trace_v2 &rhs) { + return + lhs.id == rhs.id && + lhs.number == rhs.number && + lhs.previous_id == rhs.previous_id && + lhs.timestamp == rhs.timestamp && + lhs.producer == rhs.producer && + lhs.transaction_mroot == rhs.transaction_mroot && + lhs.action_mroot == rhs.action_mroot && + lhs.schedule_version == rhs.schedule_version && + lhs.transactions == rhs.transactions; + } + std::ostream& operator<<(std::ostream &os, const block_trace_v0 &bt) { os << fc::json::to_string( bt, fc::time_point::maximum() ); return os; } + std::ostream& operator<<(std::ostream &os, const block_trace_v2 &bt) { + os << fc::json::to_string( bt, fc::time_point::maximum() ); + return os; + } + bool operator==(const block_entry_v0& lhs, const block_entry_v0& rhs) { return lhs.id == rhs.id && @@ -212,7 +180,7 @@ namespace eosio::trace_api { namespace fc { template - std::ostream& operator<<(std::ostream &os, const fc::static_variant& v ) { + std::ostream& operator<<(std::ostream &os, const std::variant& v ) { os << fc::json::to_string(v, fc::time_point::maximum()); return os; } diff --git a/plugins/trace_api_plugin/test/test_compressed_file.cpp b/plugins/trace_api_plugin/test/test_compressed_file.cpp index 66c9667cec1..9df0778d8e7 100644 --- a/plugins/trace_api_plugin/test/test_compressed_file.cpp +++ b/plugins/trace_api_plugin/test/test_compressed_file.cpp @@ -115,7 +115,7 @@ BOOST_FIXTURE_TEST_CASE_TEMPLATE(random_access_test, T, test_types, temp_file_fi BOOST_TEST(compressed_file::process(uncompressed_filename, compressed_filename, 512)); // test that you can read all of the offsets from the compressed form by opening and seeking to them - for (int i = 0; i < data.size(); i++) { + for (std::size_t i = 0; i < data.size(); i++) { const auto& entry = data.at(i); auto compf = compressed_file(compressed_filename); compf.open(); @@ -165,7 +165,7 @@ BOOST_FIXTURE_TEST_CASE_TEMPLATE(blob_access, T, test_types, temp_file_fixture) BOOST_TEST(compressed_file::process(uncompressed_filename, compressed_filename, 512)); // test that you can read all of the offsets from the compressed form through the end of the file - for (int i = 0; i < data.size(); i++) { + for (std::size_t i = 0; i < data.size(); i++) { auto actual_data = std::vector(128); auto compf = compressed_file(compressed_filename); compf.open(); @@ -201,7 +201,7 @@ BOOST_FIXTURE_TEST_CASE_TEMPLATE(blob_access_no_seek_points, T, test_types, temp BOOST_REQUIRE_EQUAL(expected_seek_point_count, actual_seek_point_count); // test that you can read all of the offsets from the compressed form through the end of the file - for (int i = 0; i < data.size(); i++) { + for (std::size_t i = 0; i < data.size(); i++) { auto actual_data = std::vector(32); auto compf = compressed_file(compressed_filename); compf.open(); diff --git a/plugins/trace_api_plugin/test/test_data_handlers.cpp b/plugins/trace_api_plugin/test/test_data_handlers.cpp index 221d45e81d9..02a63f576fc 100644 --- a/plugins/trace_api_plugin/test/test_data_handlers.cpp +++ b/plugins/trace_api_plugin/test/test_data_handlers.cpp @@ -15,12 +15,28 @@ BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) auto action = action_trace_v0 { 0, "alice"_n, "alice"_n, "foo"_n, {}, {} }; + std::variant action_trace_t = action; abi_data_handler handler(exception_handler{}); auto expected = fc::variant(); - auto actual = handler.process_data(action, [](){}); + auto actual = handler.serialize_to_variant(action_trace_t, [](){}); - BOOST_TEST(to_kv(expected) == to_kv(actual), boost::test_tools::per_element()); + BOOST_TEST(to_kv(expected) == to_kv(std::get<0>(actual)), boost::test_tools::per_element()); + } + + BOOST_AUTO_TEST_CASE(empty_data_v1) + { + auto action = action_trace_v1 { + {0, "alice"_n, "alice"_n, "foo"_n, {}, {}}, + {} + }; + std::variant action_trace_t = action; + abi_data_handler handler(exception_handler{}); + + auto expected = fc::variant(); + auto actual = handler.serialize_to_variant(action_trace_t, [](){}); + + BOOST_TEST(to_kv(expected) == to_kv(std::get<0>(actual)), boost::test_tools::per_element()); } BOOST_AUTO_TEST_CASE(no_abi) @@ -28,20 +44,38 @@ BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) auto action = action_trace_v0 { 0, "alice"_n, "alice"_n, "foo"_n, {}, {0x00, 0x01, 0x02, 0x03} }; + std::variant action_trace_t = action; + abi_data_handler handler(exception_handler{}); + + auto expected = fc::variant(); + auto actual = handler.serialize_to_variant(action_trace_t, [](){}); + + BOOST_TEST(to_kv(expected) == to_kv(std::get<0>(actual)), boost::test_tools::per_element()); + } + + BOOST_AUTO_TEST_CASE(no_abi_v1) + { + auto action = action_trace_v1 { + { 0, "alice"_n, "alice"_n, "foo"_n, {}, {0x00, 0x01, 0x02, 0x03}}, + {0x04, 0x05, 0x06, 0x07} + }; + std::variant action_trace_t = action; abi_data_handler handler(exception_handler{}); auto expected = fc::variant(); - auto actual = handler.process_data(action, [](){}); + auto actual = handler.serialize_to_variant(action_trace_t, [](){}); - BOOST_TEST(to_kv(expected) == to_kv(actual), boost::test_tools::per_element()); + BOOST_TEST(to_kv(expected) == to_kv(std::get<0>(actual)), boost::test_tools::per_element()); } BOOST_AUTO_TEST_CASE(basic_abi) { auto action = action_trace_v0 { - 0, "alice"_n, "alice"_n, "foo"_n, {}, {0x00, 0x01, 0x02, 0x03} + 0, "alice"_n, "alice"_n, "foo"_n, {}, {0x00, 0x01, 0x02, 0x03} }; + std::variant action_trace_t = action; + auto abi = chain::abi_def ( {}, { { "foo", "", { {"a", "varuint32"}, {"b", "varuint32"}, {"c", "varuint32"}, {"d", "varuint32"} } } @@ -49,7 +83,7 @@ BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) { { "foo"_n, "foo", ""} }, - {}, {}, {} + {}, {}, {}, {} ); abi.version = "eosio::abi/1."; @@ -62,17 +96,52 @@ BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) ("c", 2) ("d", 3); - auto actual = handler.process_data(action, [](){}); + auto actual = handler.serialize_to_variant(action_trace_t, [](){}); - BOOST_TEST(to_kv(expected) == to_kv(actual), boost::test_tools::per_element()); + BOOST_TEST(to_kv(expected) == to_kv(std::get<0>(actual)), boost::test_tools::per_element()); + } + + BOOST_AUTO_TEST_CASE(basic_abi_v1) + { + auto action = action_trace_v1 { + { 0, "alice"_n, "alice"_n, "foo"_n, {}, {0x00, 0x01, 0x02, 0x03}}, + {0x04, 0x05, 0x06, 0x07} + }; + std::variant action_trace_t = action; + + auto abi = chain::abi_def ( {}, + { + { "foo", "", { {"a", "varuint32"}, {"b", "varuint32"}, {"c", "varuint32"}, {"d", "varuint32"} } } + }, + { + { "foo"_n, "foo", ""} + }, + {}, {}, {}, {} + ); + abi.version = "eosio::abi/1."; + + abi_data_handler handler(exception_handler{}); + handler.add_abi("alice"_n, abi); + + fc::variant expected = fc::mutable_variant_object() + ("a", 0) + ("b", 1) + ("c", 2) + ("d", 3); + + auto actual = handler.serialize_to_variant(action_trace_t, [](){}); + + BOOST_TEST(to_kv(expected) == to_kv(std::get<0>(actual)), boost::test_tools::per_element()); } BOOST_AUTO_TEST_CASE(basic_abi_wrong_type) { auto action = action_trace_v0 { - 0, "alice"_n, "alice"_n, "foo"_n, {}, {0x00, 0x01, 0x02, 0x03} + 0, "alice"_n, "alice"_n, "foo"_n, {}, {0x00, 0x01, 0x02, 0x03} }; + std::variant action_trace_t = action; + auto abi = chain::abi_def ( {}, { { "foo", "", { {"a", "varuint32"}, {"b", "varuint32"}, {"c", "varuint32"}, {"d", "varuint32"} } } @@ -80,7 +149,7 @@ BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) { { "bar"_n, "foo", ""} }, - {}, {}, {} + {}, {}, {}, {} ); abi.version = "eosio::abi/1."; @@ -89,9 +158,38 @@ BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) auto expected = fc::variant(); - auto actual = handler.process_data(action, [](){}); + auto actual = handler.serialize_to_variant(action_trace_t, [](){}); - BOOST_TEST(to_kv(expected) == to_kv(actual), boost::test_tools::per_element()); + BOOST_TEST(to_kv(expected) == to_kv(std::get<0>(actual)), boost::test_tools::per_element()); + } + + BOOST_AUTO_TEST_CASE(basic_abi_wrong_type_v1) + { + auto action = action_trace_v1 { + { 0, "alice"_n, "alice"_n, "foo"_n, {}, {0x00, 0x01, 0x02, 0x03}}, + {0x04, 0x05, 0x06, 0x07} + }; + std::variant action_trace_t = action; + + auto abi = chain::abi_def ( {}, + { + { "foo", "", { {"a", "varuint32"}, {"b", "varuint32"}, {"c", "varuint32"}, {"d", "varuint32"} } } + }, + { + { "bar"_n, "foo", ""} + }, + {}, {}, {}, {} + ); + abi.version = "eosio::abi/1."; + + abi_data_handler handler(exception_handler{}); + handler.add_abi("alice"_n, abi); + + auto expected = fc::variant(); + + auto actual = handler.serialize_to_variant(action_trace_t, [](){}); + + BOOST_TEST(to_kv(expected) == to_kv(std::get<0>(actual)), boost::test_tools::per_element()); } BOOST_AUTO_TEST_CASE(basic_abi_insufficient_data) @@ -100,6 +198,8 @@ BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) 0, "alice"_n, "alice"_n, "foo"_n, {}, {0x00, 0x01, 0x02} }; + std::variant action_trace_t = action; + auto abi = chain::abi_def ( {}, { { "foo", "", { {"a", "varuint32"}, {"b", "varuint32"}, {"c", "varuint32"}, {"d", "varuint32"} } } @@ -107,7 +207,7 @@ BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) { { "foo"_n, "foo", ""} }, - {}, {}, {} + {}, {}, {}, {} ); abi.version = "eosio::abi/1."; @@ -117,9 +217,9 @@ BOOST_AUTO_TEST_SUITE(abi_data_handler_tests) auto expected = fc::variant(); - auto actual = handler.process_data(action, [](){}); + auto actual = handler.serialize_to_variant(action_trace_t, [](){}); - BOOST_TEST(to_kv(expected) == to_kv(actual), boost::test_tools::per_element()); + BOOST_TEST(to_kv(expected) == to_kv(std::get<0>(actual)), boost::test_tools::per_element()); BOOST_TEST(log_called); } diff --git a/plugins/trace_api_plugin/test/test_extraction.cpp b/plugins/trace_api_plugin/test/test_extraction.cpp index 21e213f7bcb..dbbfae8fe95 100644 --- a/plugins/trace_api_plugin/test/test_extraction.cpp +++ b/plugins/trace_api_plugin/test/test_extraction.cpp @@ -5,10 +5,14 @@ #include #include #include +#include +#include #include #include +#include + using namespace eosio; using namespace eosio::trace_api; using namespace eosio::trace_api::test_common; @@ -48,6 +52,15 @@ namespace { return result; } + auto get_private_key( name keyname, std::string role = "owner" ) { + auto secret = fc::sha256::hash( keyname.to_string() + role ); + return chain::private_key_type::regenerate( secret ); + } + + auto get_public_key( name keyname, std::string role = "owner" ) { + return get_private_key( keyname, role ).get_public_key(); + } + auto make_transfer_action( chain::name from, chain::name to, chain::asset quantity, std::string memo ) { return chain::action( std::vector {{from, chain::config::active_name}}, "eosio.token"_n, "transfer"_n, make_transfer_data( from, to, quantity, std::move(memo) ) ); @@ -61,7 +74,7 @@ namespace { auto make_packed_trx( std::vector actions ) { chain::signed_transaction trx; trx.actions = std::move( actions ); - return packed_transaction( trx ); + return packed_transaction( std::move(trx), true ); } auto make_trx_header( const chain::transaction& trx ) { @@ -92,6 +105,60 @@ namespace { return result; } + auto make_block_state( chain::block_id_type previous, uint32_t height, uint32_t slot, chain::name producer, + std::vector trxs ) { + chain::signed_block_ptr block = std::make_shared(); + for( auto& trx : trxs ) { + block->transactions.emplace_back( trx ); + } + block->producer = producer; + block->timestamp = chain::block_timestamp_type(slot); + // make sure previous contains correct block # so block_header::block_num() returns correct value + if( previous == chain::block_id_type() ) { + previous._hash[0] &= 0xffffffff00000000; + previous._hash[0] += fc::endian_reverse_u32(height - 1); + } + block->previous = previous; + + auto priv_key = get_private_key( block->producer, "active" ); + auto pub_key = get_public_key( block->producer, "active" ); + + auto prev = std::make_shared(); + auto header_bmroot = digest_type::hash( std::make_pair( block->digest(), prev->blockroot_merkle.get_root())); + auto sig_digest = digest_type::hash( std::make_pair( header_bmroot, prev->pending_schedule.schedule_hash )); + block->producer_signature = priv_key.sign( sig_digest ); + + std::vector signing_keys; + signing_keys.emplace_back( std::move( priv_key )); + auto signer = [&]( digest_type d ) { + std::vector result; + result.reserve( signing_keys.size()); + for( const auto& k: signing_keys ) + result.emplace_back( k.sign( d )); + return result; + }; + chain::pending_block_header_state pbhs; + pbhs.producer = block->producer; + pbhs.timestamp = block->timestamp; + chain::producer_authority_schedule schedule = {0, {chain::producer_authority{block->producer, + chain::block_signing_authority_v0{1, {{pub_key, 1}}}}}}; + pbhs.active_schedule = schedule; + pbhs.valid_block_signing_authority = chain::block_signing_authority_v0{1, {{pub_key, 1}}}; + auto bsp = std::make_shared( + std::move( pbhs ), + std::move( block ), + eosio::chain::deque(), + chain::protocol_feature_set(), + []( chain::block_timestamp_type timestamp, + const fc::flat_set& cur_features, + const std::vector& new_features ) {}, + signer + ); + bsp->block_num = height; + + return bsp; + } + } struct extraction_test_fixture { @@ -108,11 +175,8 @@ struct extraction_test_fixture { * * @param entry : the entry to append */ - void append( const block_trace_v0& entry ) { - fixture.data_log.emplace_back(entry); - } - - void append( const block_trace_v1& entry ) { + template + void append( const BlockTrace& entry ) { fixture.data_log.emplace_back(entry); } @@ -128,8 +192,8 @@ struct extraction_test_fixture { { } - void signal_applied_transaction( const chain::transaction_trace_ptr& trace, const chain::signed_transaction& strx ) { - extraction_impl.signal_applied_transaction(trace, strx); + void signal_applied_transaction( const chain::transaction_trace_ptr& trace, const chain::packed_transaction_ptr& ptrx ) { + extraction_impl.signal_applied_transaction(trace, ptrx); } void signal_accepted_block( const chain::block_state_ptr& bsp ) { @@ -160,7 +224,7 @@ BOOST_AUTO_TEST_SUITE(block_extraction) signal_applied_transaction( make_transaction_trace( ptrx1.id(), 1, 1, chain::transaction_receipt_header::executed, { actt1, actt2, actt3 } ), - ptrx1.get_signed_transaction() ); + std::make_shared(ptrx1) ); // accept the block with one transaction auto bsp1 = make_block_state( chain::block_id_type(), 1, 1, "bp.one"_n, @@ -168,55 +232,65 @@ BOOST_AUTO_TEST_SUITE(block_extraction) signal_accepted_block( bsp1 ); const uint32_t expected_lib = 0; - const block_trace_v1 expected_trace{ + + const std::vector expected_action_traces { { - bsp1->id, - 1, - bsp1->prev(), - chain::block_timestamp_type(1), - "bp.one"_n + { + 0, + "eosio.token"_n, "eosio.token"_n, "transfer"_n, + {{"alice"_n, "active"_n}}, + make_transfer_data("alice"_n, "bob"_n, "0.0001 SYS"_t, "Memo!") + }, + {} }, + { + { + 1, + "alice"_n, "eosio.token"_n, "transfer"_n, + {{"alice"_n, "active"_n}}, + make_transfer_data("alice"_n, "bob"_n, "0.0001 SYS"_t, "Memo!") + }, + {} + }, + { + { + 2, + "bob"_n, "eosio.token"_n, "transfer"_n, + {{"alice"_n, "active"_n}}, + make_transfer_data("alice"_n, "bob"_n, "0.0001 SYS"_t, "Memo!") + }, + {} + } + }; + + const transaction_trace_v2 expected_transaction_trace { + ptrx1.id(), + expected_action_traces, + fc::enum_type{bsp1->block->transactions[0].status}, + bsp1->block->transactions[0].cpu_usage_us, + bsp1->block->transactions[0].net_usage_words, + *ptrx1.get_signatures(), + make_trx_header(ptrx1.get_transaction()) + }; + + const block_trace_v2 expected_block_trace { + bsp1->id, + 1, + bsp1->prev(), + chain::block_timestamp_type(1), + "bp.one"_n, bsp1->block->transaction_mroot, bsp1->block->action_mroot, bsp1->block->schedule_version, - { - { - { - ptrx1.id(), - { - { - 0, - "eosio.token"_n, "eosio.token"_n, "transfer"_n, - {{ "alice"_n, "active"_n }}, - make_transfer_data( "alice"_n, "bob"_n, "0.0001 SYS"_t, "Memo!" ) - }, - { - 1, - "alice"_n, "eosio.token"_n, "transfer"_n, - {{ "alice"_n, "active"_n }}, - make_transfer_data( "alice"_n, "bob"_n, "0.0001 SYS"_t, "Memo!" ) - }, - { - 2, - "bob"_n, "eosio.token"_n, "transfer"_n, - {{ "alice"_n, "active"_n }}, - make_transfer_data( "alice"_n, "bob"_n, "0.0001 SYS"_t, "Memo!" ) - } - } - }, - fc::enum_type{bsp1->block->transactions[0].status}, - bsp1->block->transactions[0].cpu_usage_us, - bsp1->block->transactions[0].net_usage_words, - ptrx1.get_signatures(), - make_trx_header(ptrx1.get_transaction()) - } + std::vector { + expected_transaction_trace } }; BOOST_REQUIRE_EQUAL(max_lib, 0); BOOST_REQUIRE(data_log.size() == 1); - BOOST_REQUIRE(data_log.at(0).contains()); - BOOST_REQUIRE_EQUAL(data_log.at(0).get(), expected_trace); + BOOST_REQUIRE(std::holds_alternative(data_log.at(0))); + BOOST_REQUIRE_EQUAL(std::get(data_log.at(0)), expected_block_trace); } BOOST_FIXTURE_TEST_CASE(basic_multi_transaction_block, extraction_test_fixture) { @@ -233,15 +307,15 @@ BOOST_AUTO_TEST_SUITE(block_extraction) signal_applied_transaction( make_transaction_trace( ptrx1.id(), 1, 1, chain::transaction_receipt_header::executed, { actt1 } ), - ptrx1.get_signed_transaction() ); + std::make_shared( ptrx1 ) ); signal_applied_transaction( make_transaction_trace( ptrx2.id(), 1, 1, chain::transaction_receipt_header::executed, { actt2 } ), - ptrx2.get_signed_transaction() ); + std::make_shared( ptrx2 ) ); signal_applied_transaction( make_transaction_trace( ptrx3.id(), 1, 1, chain::transaction_receipt_header::executed, { actt3 } ), - ptrx3.get_signed_transaction() ); + std::make_shared( ptrx3 ) ); // accept the block with three transaction auto bsp1 = make_block_state( chain::block_id_type(), 1, 1, "bp.one"_n, @@ -250,81 +324,88 @@ BOOST_AUTO_TEST_SUITE(block_extraction) const uint32_t expected_lib = 0; - const block_trace_v1 expected_trace{ - { - bsp1->id, - 1, - bsp1->prev(), - chain::block_timestamp_type(1), - "bp.one"_n - }, - bsp1->block->transaction_mroot, - bsp1->block->action_mroot, - bsp1->block->schedule_version, + const std::vector expected_action_trace1 { { { - { - ptrx1.id(), - { - { - 0, - "eosio.token"_n, "eosio.token"_n, "transfer"_n, - {{ "alice"_n, "active"_n }}, - make_transfer_data( "alice"_n, "bob"_n, "0.0001 SYS"_t, "Memo!" ) - } - } - }, - fc::enum_type{bsp1->block->transactions[0].status}, - bsp1->block->transactions[0].cpu_usage_us, - bsp1->block->transactions[0].net_usage_words, - ptrx1.get_signatures(), - make_trx_header(ptrx1.get_transaction()) - } - , + 0, + "eosio.token"_n, "eosio.token"_n, "transfer"_n, + {{"alice"_n, "active"_n}}, + make_transfer_data("alice"_n, "bob"_n, "0.0001 SYS"_t, "Memo!") + }, + {} + } + }; + + const std::vector expected_action_trace2 { + { { - { - ptrx2.id(), - { - { - 1, - "bob"_n, "eosio.token"_n, "transfer"_n, - {{ "bob"_n, "active"_n }}, - make_transfer_data( "bob"_n, "alice"_n, "0.0001 SYS"_t, "Memo!" ) - } - } - }, - fc::enum_type{bsp1->block->transactions[1].status}, - bsp1->block->transactions[1].cpu_usage_us, - bsp1->block->transactions[1].net_usage_words, - ptrx2.get_signatures(), - make_trx_header(ptrx2.get_transaction()) - } - , + 1, + "bob"_n, "eosio.token"_n, "transfer"_n, + {{ "bob"_n, "active"_n }}, + make_transfer_data( "bob"_n, "alice"_n, "0.0001 SYS"_t, "Memo!" ) + }, + {} + } + }; + + const std::vector expected_action_trace3 { + { { - { - ptrx3.id(), - { - { - 2, - "fred"_n, "eosio.token"_n, "transfer"_n, - {{ "fred"_n, "active"_n }}, - make_transfer_data( "fred"_n, "bob"_n, "0.0001 SYS"_t, "Memo!" ) - } - } - }, - fc::enum_type{bsp1->block->transactions[2].status}, - bsp1->block->transactions[2].cpu_usage_us, - bsp1->block->transactions[2].net_usage_words, - ptrx3.get_signatures(), - make_trx_header(ptrx3.get_transaction()) - } + 2, + "fred"_n, "eosio.token"_n, "transfer"_n, + {{ "fred"_n, "active"_n }}, + make_transfer_data( "fred"_n, "bob"_n, "0.0001 SYS"_t, "Memo!" ) + }, + {} } }; + const std::vector expected_transaction_traces { + { + ptrx1.id(), + expected_action_trace1, + fc::enum_type{bsp1->block->transactions[0].status}, + bsp1->block->transactions[0].cpu_usage_us, + bsp1->block->transactions[0].net_usage_words, + *ptrx1.get_signatures(), + make_trx_header(ptrx1.get_transaction()) + }, + { + ptrx2.id(), + expected_action_trace2, + fc::enum_type{bsp1->block->transactions[1].status}, + bsp1->block->transactions[1].cpu_usage_us, + bsp1->block->transactions[1].net_usage_words, + *ptrx2.get_signatures(), + make_trx_header(ptrx2.get_transaction()) + }, + { + ptrx3.id(), + expected_action_trace3, + fc::enum_type{bsp1->block->transactions[2].status}, + bsp1->block->transactions[2].cpu_usage_us, + bsp1->block->transactions[2].net_usage_words, + *ptrx3.get_signatures(), + make_trx_header(ptrx3.get_transaction()) + } + }; + + const block_trace_v2 expected_block_trace{ + bsp1->id, + 1, + bsp1->prev(), + chain::block_timestamp_type(1), + "bp.one"_n, + bsp1->block->transaction_mroot, + bsp1->block->action_mroot, + bsp1->block->schedule_version, + expected_transaction_traces + }; + BOOST_REQUIRE_EQUAL(max_lib, 0); BOOST_REQUIRE(data_log.size() == 1); - BOOST_REQUIRE(data_log.at(0).contains()); - BOOST_REQUIRE_EQUAL(data_log.at(0).get(), expected_trace); + BOOST_REQUIRE(std::holds_alternative(data_log.at(0))); + BOOST_REQUIRE_EQUAL(std::get(data_log.at(0)), expected_block_trace); } BOOST_FIXTURE_TEST_CASE(onerror_transaction_block, extraction_test_fixture) @@ -343,50 +424,54 @@ BOOST_AUTO_TEST_SUITE(block_extraction) { actt2 } ); onerror_trace->failed_dtrx_trace = transfer_trace; - signal_applied_transaction( onerror_trace, transfer_trx.get_signed_transaction() ); + signal_applied_transaction( onerror_trace, std::make_shared( transfer_trx ) ); auto bsp1 = make_block_state( chain::block_id_type(), 1, 1, "bp.one"_n, { chain::packed_transaction(transfer_trx) } ); signal_accepted_block( bsp1 ); const uint32_t expected_lib = 0; - const block_trace_v1 expected_trace { + + const std::vector expected_action_trace { { - bsp1->id, - 1, - bsp1->prev(), - chain::block_timestamp_type(1), - "bp.one"_n - }, + { + 0, + "eosio.token"_n, "eosio"_n, "onerror"_n, + {{ "alice"_n, "active"_n }}, + make_onerror_data( chain::onerror{ 1, "test ", 4 } ) + }, + {} + } + }; + + const std::vector expected_transaction_traces { + { + transfer_trx.id(), // transfer_trx.id() because that is the trx id known to the user + expected_action_trace, + fc::enum_type{bsp1->block->transactions[0].status}, + bsp1->block->transactions[0].cpu_usage_us, + bsp1->block->transactions[0].net_usage_words, + *transfer_trx.get_signatures(), + make_trx_header(transfer_trx.get_transaction()) + } + }; + + const block_trace_v2 expected_block_trace { + bsp1->id, + 1, + bsp1->prev(), + chain::block_timestamp_type(1), + "bp.one"_n, bsp1->block->transaction_mroot, bsp1->block->action_mroot, bsp1->block->schedule_version, - { - { - { - transfer_trx.id(), // transfer_trx.id() because that is the trx id known to the user - { - { - 0, - "eosio.token"_n, "eosio"_n, "onerror"_n, - {{ "alice"_n, "active"_n }}, - make_onerror_data( chain::onerror{ 1, "test ", 4 } ) - } - } - }, - fc::enum_type{bsp1->block->transactions[0].status}, - bsp1->block->transactions[0].cpu_usage_us, - bsp1->block->transactions[0].net_usage_words, - transfer_trx.get_signatures(), - make_trx_header(transfer_trx.get_transaction()) - } - } + expected_transaction_traces }; BOOST_REQUIRE_EQUAL(max_lib, 0); BOOST_REQUIRE(data_log.size() == 1); - BOOST_REQUIRE(data_log.at(0).contains()); - BOOST_REQUIRE_EQUAL(data_log.at(0).get(), expected_trace); + BOOST_REQUIRE(std::holds_alternative(data_log.at(0))); + BOOST_REQUIRE_EQUAL(std::get(data_log.at(0)), expected_block_trace); } BOOST_AUTO_TEST_SUITE_END() diff --git a/plugins/trace_api_plugin/test/test_responses.cpp b/plugins/trace_api_plugin/test/test_responses.cpp index 33980bea524..50673141de2 100644 --- a/plugins/trace_api_plugin/test/test_responses.cpp +++ b/plugins/trace_api_plugin/test/test_responses.cpp @@ -32,18 +32,27 @@ struct response_test_fixture { response_test_fixture& fixture; }; - constexpr static auto default_mock_data_handler = [](const action_trace_v0& a, const yield_function&) -> fc::variant { - return fc::mutable_variant_object()("hex" , fc::to_hex(a.data.data(), a.data.size())); + constexpr static auto default_mock_data_handler_v0 = [](const action_trace_v0& a, const yield_function&) ->std::tuple> { + return {fc::mutable_variant_object()("hex" , fc::to_hex(a.data.data(), a.data.size())),{}}; }; + constexpr static auto default_mock_data_handler_v1 = [](const action_trace_v1& a, const yield_function&) -> std::tuple>{ + return {fc::mutable_variant_object()("hex" , fc::to_hex(a.data.data(), a.data.size())), {fc::mutable_variant_object()("hex" , fc::to_hex(a.return_value.data(), a.return_value.size()))}}; + }; struct mock_data_handler_provider { mock_data_handler_provider(response_test_fixture& fixture) :fixture(fixture) {} - fc::variant process_data(const action_trace_v0& action, const yield_function& yield) { - return fixture.mock_data_handler(action, yield); + template + std::tuple> serialize_to_variant(const ActionTrace & action, const yield_function& yield) { + if constexpr(std::is_same_v){ + return fixture.mock_data_handler_v0(action, yield); + } + else if constexpr(std::is_same_v){ + return fixture.mock_data_handler_v1(action, yield); + } } response_test_fixture& fixture; @@ -65,7 +74,8 @@ struct response_test_fixture { // fixture data and methods std::function mock_get_block; - std::function mock_data_handler = default_mock_data_handler; + std::function>(const action_trace_v0&, const yield_function&)> mock_data_handler_v0 = default_mock_data_handler_v0; + std::function>(const action_trace_v1&, const yield_function&)> mock_data_handler_v1 = default_mock_data_handler_v1; response_impl_type response_impl; @@ -113,6 +123,25 @@ BOOST_AUTO_TEST_SUITE(trace_responses) BOOST_FIXTURE_TEST_CASE(basic_block_response, response_test_fixture) { + auto action_trace = action_trace_v0 { + 0, + "receiver"_n, "contract"_n, "action"_n, + {{ "alice"_n, "active"_n }}, + { 0x00, 0x01, 0x02, 0x03 } + }; + + auto transaction_trace = transaction_trace_v1 { { + "0000000000000000000000000000000000000000000000000000000000000001"_h, + { + action_trace + }}, + fc::enum_type{chain::transaction_receipt_header::status_enum::executed}, + 10, + 5, + std::vector{ chain::signature_type() }, + { chain::time_point(), 1, 0, 100, 50, 0 } + }; + auto block_trace = block_trace_v1 { { "b000000000000000000000000000000000000000000000000000000000000001"_h, @@ -125,24 +154,7 @@ BOOST_AUTO_TEST_SUITE(trace_responses) "0000000000000000000000000000000000000000000000000000000000000000"_h, 0, { - { - { - "0000000000000000000000000000000000000000000000000000000000000001"_h, - { - { - 0, - "receiver"_n, "contract"_n, "action"_n, - {{ "alice"_n, "active"_n }}, - { 0x00, 0x01, 0x02, 0x03 } - } - } - }, - fc::enum_type{chain::transaction_receipt_header::status_enum::executed}, - 10, - 5, - std::vector{ chain::signature_type() }, - { chain::time_point(), 1, 0, 100, 50, 0 } - } + transaction_trace } }; @@ -281,7 +293,7 @@ BOOST_AUTO_TEST_SUITE(trace_responses) }; // simulate an inability to parse the parameters - mock_data_handler = [](const action_trace_v0&, const yield_function&) -> fc::variant { + mock_data_handler_v0 = [](const action_trace_v0&, const yield_function&) -> std::tuple> { return {}; }; @@ -408,7 +420,7 @@ BOOST_AUTO_TEST_SUITE(trace_responses) }; // simulate an inability to parse the parameters - mock_data_handler = [](const action_trace_v0&, const yield_function&) -> fc::variant { + mock_data_handler_v0 = [](const action_trace_v0&, const yield_function&) -> std::tuple> { return {}; }; @@ -605,4 +617,457 @@ BOOST_AUTO_TEST_SUITE(trace_responses) BOOST_TEST(to_kv(expected_response) == to_kv(actual_response), boost::test_tools::per_element()); } + BOOST_FIXTURE_TEST_CASE(basic_empty_block_response_v2, response_test_fixture) + { + auto block_trace = block_trace_v2 { + "b000000000000000000000000000000000000000000000000000000000000001"_h, // block id + 1, + "0000000000000000000000000000000000000000000000000000000000000000"_h, // previous id + chain::block_timestamp_type(0), + "bp.one"_n, + "0000000000000000000000000000000000000000000000000000000000000000"_h, // transaction mroot + "0000000000000000000000000000000000000000000000000000000000000000"_h, // action mroot + 0, // schedule version + {} // transactions + }; + + fc::variant expected_response = fc::mutable_variant_object() + ("id", "b000000000000000000000000000000000000000000000000000000000000001") + ("number", 1) + ("previous_id", "0000000000000000000000000000000000000000000000000000000000000000") + ("status", "pending") + ("timestamp", "2000-01-01T00:00:00.000Z") + ("producer", "bp.one") + ("transaction_mroot", "0000000000000000000000000000000000000000000000000000000000000000") + ("action_mroot", "0000000000000000000000000000000000000000000000000000000000000000") + ("schedule_version", 0) + ("transactions", fc::variants() ) + ; + + mock_get_block = [&block_trace]( uint32_t height, const yield_function& ) -> get_block_t { + BOOST_TEST(height == 1); + return std::make_tuple(data_log_entry{block_trace}, false); + }; + + fc::variant actual_response = get_block_trace( 1 ); + + BOOST_TEST(to_kv(expected_response) == to_kv(actual_response), boost::test_tools::per_element()); + } + + BOOST_FIXTURE_TEST_CASE(basic_block_response_v2, response_test_fixture) + { + auto action_trace = action_trace_v1 { + { + 0, + "receiver"_n, "contract"_n, "action"_n, + {{ "alice"_n, "active"_n }}, + { 0x00, 0x01, 0x02, 0x03 } + }, + { 0x04, 0x05, 0x06, 0x07 } + }; + + auto transaction_trace = transaction_trace_v2 {//trn + "0000000000000000000000000000000000000000000000000000000000000001"_h, + std::vector { + action_trace + }, + fc::enum_type{chain::transaction_receipt_header::status_enum::executed}, + 10, // cpu_usage_us + 5, // net_usage_words + std::vector{ chain::signature_type() }, // signatures + { chain::time_point(), 1, 0, 100, 50, 0 } // trx_header + };// trn end + + auto block_trace = block_trace_v2 { + "b000000000000000000000000000000000000000000000000000000000000001"_h, // block id + 1, + "0000000000000000000000000000000000000000000000000000000000000000"_h, // previous id + chain::block_timestamp_type(0), + "bp.one"_n, + "0000000000000000000000000000000000000000000000000000000000000000"_h, // transaction mroot + "0000000000000000000000000000000000000000000000000000000000000000"_h, //action mroot + 0, // schedule version + std::vector { + transaction_trace + } + }; + + fc::variant expected_response = fc::mutable_variant_object() + ("id", "b000000000000000000000000000000000000000000000000000000000000001") + ("number", 1) + ("previous_id", "0000000000000000000000000000000000000000000000000000000000000000") + ("status", "pending") + ("timestamp", "2000-01-01T00:00:00.000Z") + ("producer", "bp.one") + ("transaction_mroot", "0000000000000000000000000000000000000000000000000000000000000000") + ("action_mroot", "0000000000000000000000000000000000000000000000000000000000000000") + ("schedule_version", 0) + ("transactions", fc::variants({ + fc::mutable_variant_object() + ("id", "0000000000000000000000000000000000000000000000000000000000000001") + ("actions", fc::variants({ + fc::mutable_variant_object() + ("global_sequence", 0) + ("receiver", "receiver") + ("account", "contract") + ("action", "action") + ("authorization", fc::variants({ + fc::mutable_variant_object() + ("account", "alice") + ("permission", "active") + })) + ("data", "00010203") + ("return_value", "04050607") + ("params", fc::mutable_variant_object() + ("hex", "00010203")) + ("return_data", fc::mutable_variant_object() + ("hex", "04050607")) + })) + ("status", "executed") + ("cpu_usage_us", 10) + ("net_usage_words", 5) + ("signatures", fc::variants({"SIG_K1_111111111111111111111111111111111111111111111111111111111111111116uk5ne"})) + ("transaction_header", fc::mutable_variant_object() + ("expiration", "1970-01-01T00:00:00") + ("ref_block_num", 1) + ("ref_block_prefix", 0) + ("max_net_usage_words", 100) + ("max_cpu_usage_ms", 50) + ("delay_sec", 0) + ) + })) + ; + + mock_get_block = [&block_trace]( uint32_t height, const yield_function& ) -> get_block_t { + BOOST_TEST(height == 1); + return std::make_tuple(data_log_entry(block_trace), false); + }; + + fc::variant actual_response = get_block_trace( 1 ); + BOOST_TEST(to_kv(expected_response) == to_kv(actual_response), boost::test_tools::per_element()); + } + + BOOST_FIXTURE_TEST_CASE(basic_block_response_no_params_v2, response_test_fixture) + { + auto action_trace = action_trace_v1 { + { + 0, + "receiver"_n, "contract"_n, "action"_n, + {{ "alice"_n, "active"_n }}, + { 0x00, 0x01, 0x02, 0x03 } + }, + { 0x04, 0x05, 0x06, 0x07 } + }; + + auto transaction_trace = transaction_trace_v2 { + "0000000000000000000000000000000000000000000000000000000000000001"_h, + std::vector { + action_trace + }, + fc::enum_type{chain::transaction_receipt_header::status_enum::executed}, + 10, + 5, + std::vector{ chain::signature_type() }, + { chain::time_point(), 1, 0, 100, 50, 0 } + }; + + auto block_trace = block_trace_v2 { + "b000000000000000000000000000000000000000000000000000000000000001"_h, + 1, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + chain::block_timestamp_type(0), + "bp.one"_n, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + 0, + std::vector { + transaction_trace + } + }; + + fc::variant expected_response = fc::mutable_variant_object() + ("id", "b000000000000000000000000000000000000000000000000000000000000001") + ("number", 1) + ("previous_id", "0000000000000000000000000000000000000000000000000000000000000000") + ("status", "pending") + ("timestamp", "2000-01-01T00:00:00.000Z") + ("producer", "bp.one") + ("transaction_mroot", "0000000000000000000000000000000000000000000000000000000000000000") + ("action_mroot", "0000000000000000000000000000000000000000000000000000000000000000") + ("schedule_version", 0) + ("transactions", fc::variants({ + fc::mutable_variant_object() + ("id", "0000000000000000000000000000000000000000000000000000000000000001") + ("actions", fc::variants({ + fc::mutable_variant_object() + ("global_sequence", 0) + ("receiver", "receiver") + ("account", "contract") + ("action", "action") + ("authorization", fc::variants({ + fc::mutable_variant_object() + ("account", "alice") + ("permission", "active") + })) + ("data", "00010203") + ("return_value", "04050607") + })) + ("status", "executed") + ("cpu_usage_us", 10) + ("net_usage_words", 5) + ("signatures", fc::variants({"SIG_K1_111111111111111111111111111111111111111111111111111111111111111116uk5ne"})) + ("transaction_header", fc::mutable_variant_object() + ("expiration", "1970-01-01T00:00:00") + ("ref_block_num", 1) + ("ref_block_prefix", 0) + ("max_net_usage_words", 100) + ("max_cpu_usage_ms", 50) + ("delay_sec", 0)) + }) + ); + + mock_get_block = [&block_trace]( uint32_t height, const yield_function& ) -> get_block_t { + BOOST_TEST(height == 1); + return std::make_tuple(data_log_entry(block_trace), false); + }; + + // simulate an inability to parse the parameters and return_data + mock_data_handler_v1 = [](const action_trace_v1&, const yield_function&) -> std::tuple> { + return {}; + }; + + fc::variant actual_response = get_block_trace( 1 ); + + BOOST_TEST(to_kv(expected_response) == to_kv(actual_response), boost::test_tools::per_element()); + } + + BOOST_FIXTURE_TEST_CASE(basic_block_response_unsorted_v2, response_test_fixture) + { + std::vector actions = { + { + { + 1, + "receiver"_n, "contract"_n, "action"_n, + {{ "alice"_n, "active"_n }}, + { 0x01, 0x01, 0x01, 0x01 }, + }, + { 0x05, 0x05, 0x05, 0x05 } + }, + { + { + 0, + "receiver"_n, "contract"_n, "action"_n, + {{ "alice"_n, "active"_n }}, + { 0x00, 0x00, 0x00, 0x00 } + }, + { 0x04, 0x04, 0x04, 0x04} + }, + { + { + 2, + "receiver"_n, "contract"_n, "action"_n, + {{ "alice"_n, "active"_n }}, + { 0x02, 0x02, 0x02, 0x02 } + }, + { 0x06, 0x06, 0x06, 0x06 } + } + }; + + auto transaction_trace = transaction_trace_v2 { + "0000000000000000000000000000000000000000000000000000000000000001"_h, + actions, + fc::enum_type{chain::transaction_receipt_header::status_enum::executed}, + 10, + 5, + { chain::signature_type() }, + { chain::time_point(), 1, 0, 100, 50, 0 } + }; + + auto block_trace = block_trace_v2 { + "b000000000000000000000000000000000000000000000000000000000000001"_h, + 1, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + chain::block_timestamp_type(0), + "bp.one"_n, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + 0, + std::vector { + transaction_trace + } + }; + + fc::variant expected_response = fc::mutable_variant_object() + ("id", "b000000000000000000000000000000000000000000000000000000000000001") + ("number", 1) + ("previous_id", "0000000000000000000000000000000000000000000000000000000000000000") + ("status", "pending") + ("timestamp", "2000-01-01T00:00:00.000Z") + ("producer", "bp.one") + ("transaction_mroot", "0000000000000000000000000000000000000000000000000000000000000000") + ("action_mroot", "0000000000000000000000000000000000000000000000000000000000000000") + ("schedule_version", 0) + ("transactions", fc::variants({ + fc::mutable_variant_object() + ("id", "0000000000000000000000000000000000000000000000000000000000000001") + ("actions", fc::variants({ + fc::mutable_variant_object() + ("global_sequence", 0) + ("receiver", "receiver") + ("account", "contract") + ("action", "action") + ("authorization", fc::variants({ + fc::mutable_variant_object() + ("account", "alice") + ("permission", "active") + })) + ("data", "00000000") + ("return_value","04040404") + , + fc::mutable_variant_object() + ("global_sequence", 1) + ("receiver", "receiver") + ("account", "contract") + ("action", "action") + ("authorization", fc::variants({ + fc::mutable_variant_object() + ("account", "alice") + ("permission", "active") + })) + ("data", "01010101") + ("return_value", "05050505") + , + fc::mutable_variant_object() + ("global_sequence", 2) + ("receiver", "receiver") + ("account", "contract") + ("action", "action") + ("authorization", fc::variants({ + fc::mutable_variant_object() + ("account", "alice") + ("permission", "active") + })) + ("data", "02020202") + ("return_value", "06060606") + })) + ("status", "executed") + ("cpu_usage_us", 10) + ("net_usage_words", 5) + ("signatures", fc::variants({"SIG_K1_111111111111111111111111111111111111111111111111111111111111111116uk5ne"})) + ("transaction_header", fc::mutable_variant_object() + ("expiration", "1970-01-01T00:00:00") + ("ref_block_num", 1) + ("ref_block_prefix", 0) + ("max_net_usage_words", 100) + ("max_cpu_usage_ms", 50) + ("delay_sec", 0) + ) + })) + ; + + mock_get_block = [&block_trace]( uint32_t height, const yield_function& ) -> get_block_t { + BOOST_TEST(height == 1); + return std::make_tuple(data_log_entry(block_trace), false); + }; + + // simulate an inability to parse the parameters and return_data + mock_data_handler_v1 = [](const action_trace_v1&, const yield_function&) -> std::tuple> { + return {}; + }; + + fc::variant actual_response = get_block_trace( 1 ); + + BOOST_TEST(to_kv(expected_response) == to_kv(actual_response), boost::test_tools::per_element()); + } + + BOOST_FIXTURE_TEST_CASE(lib_response_v2, response_test_fixture) + { + auto block_trace = block_trace_v2 { + "b000000000000000000000000000000000000000000000000000000000000001"_h, + 1, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + chain::block_timestamp_type(0), + "bp.one"_n, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + 0, + {} + }; + + fc::variant expected_response = fc::mutable_variant_object() + ("id", "b000000000000000000000000000000000000000000000000000000000000001") + ("number", 1) + ("previous_id", "0000000000000000000000000000000000000000000000000000000000000000") + ("status", "irreversible") + ("timestamp", "2000-01-01T00:00:00.000Z") + ("producer", "bp.one") + ("transaction_mroot", "0000000000000000000000000000000000000000000000000000000000000000") + ("action_mroot", "0000000000000000000000000000000000000000000000000000000000000000") + ("schedule_version", 0) + ("transactions", fc::variants() ) + ; + + mock_get_block = [&block_trace]( uint32_t height, const yield_function& ) -> get_block_t { + BOOST_TEST(height == 1); + return std::make_tuple(data_log_entry(block_trace), true); + }; + + fc::variant response = get_block_trace( 1 ); + BOOST_TEST(to_kv(expected_response) == to_kv(response), boost::test_tools::per_element()); + + } + + BOOST_FIXTURE_TEST_CASE(yield_throws_v2, response_test_fixture) + { + auto action_trace = action_trace_v1 { + { + 0, + "receiver"_n, "contract"_n, "action"_n, + {{ "alice"_n, "active"_n }}, + { 0x00, 0x01, 0x02, 0x03 } + }, + { 0x04, 0x05, 0x06, 0x07 } + }; + + auto transaction_trace = transaction_trace_v2 { + "0000000000000000000000000000000000000000000000000000000000000001"_h, + std::vector { + action_trace + }, + fc::enum_type{chain::transaction_receipt_header::status_enum::executed}, + 10, + 5, + std::vector{chain::signature_type()}, + {chain::time_point(), 1, 0, 100, 50, 0} + }; + + auto block_trace = block_trace_v2 { + "b000000000000000000000000000000000000000000000000000000000000001"_h, + 1, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + chain::block_timestamp_type(0), + "bp.one"_n, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + 0, + std::vector{ + transaction_trace + } + }; + + mock_get_block = [&block_trace]( uint32_t height, const yield_function& ) -> get_block_t { + BOOST_TEST(height == 1); + return std::make_tuple(data_log_entry(block_trace), false); + }; + + int countdown = 3; + yield_function yield = [&]() { + if (countdown-- == 0) { + throw yield_exception("mock"); + } + }; + + BOOST_REQUIRE_THROW(get_block_trace( 1, yield ), yield_exception); + } + BOOST_AUTO_TEST_SUITE_END() diff --git a/plugins/trace_api_plugin/test/test_trace_file.cpp b/plugins/trace_api_plugin/test/test_trace_file.cpp index c1f6fee077a..322c2d54bc6 100644 --- a/plugins/trace_api_plugin/test/test_trace_file.cpp +++ b/plugins/trace_api_plugin/test/test_trace_file.cpp @@ -14,7 +14,75 @@ using open_state = slice_directory::open_state; namespace { struct test_fixture { - const block_trace_v1 bt { + std::vector actions = { + { + { + 1, + "receiver"_n, "contract"_n, "action"_n, + {{ "alice"_n, "active"_n }}, + { 0x01, 0x01, 0x01, 0x01 } + }, + { 0x05, 0x05, 0x05, 0x05 } + }, + { + { + 0, + "receiver"_n, "contract"_n, "action"_n, + {{ "alice"_n, "active"_n }}, + { 0x00, 0x00, 0x00, 0x00 } + }, + { 0x04, 0x04, 0x04, 0x04} + }, + { + { + 2, + "receiver"_n, "contract"_n, "action"_n, + {{ "alice"_n, "active"_n }}, + { 0x02, 0x02, 0x02, 0x02 } + }, + { 0x06, 0x06, 0x06, 0x06 } + } + }; + + transaction_trace_v2 transaction_trace { + "0000000000000000000000000000000000000000000000000000000000000001"_h, + actions, + fc::enum_type{chain::transaction_receipt_header::status_enum::executed}, + 10, + 5, + { chain::signature_type() }, + { chain::time_point(), 1, 0, 100, 50, 0 } + }; + + block_trace_v2 block_trace1_v2 { + "b000000000000000000000000000000000000000000000000000000000000001"_h, + 1, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + chain::block_timestamp_type(0), + "bp.one"_n, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + 0, + std::vector { + transaction_trace + } + }; + + block_trace_v2 block_trace2_v2 { + "b000000000000000000000000000000000000000000000000000000000000001"_h, + 5, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + chain::block_timestamp_type(0), + "bp.one"_n, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + "0000000000000000000000000000000000000000000000000000000000000000"_h, + 0, + std::vector { + transaction_trace + } + }; + + const block_trace_v1 bt_v1 { { "0000000000000000000000000000000000000000000000000000000000000001"_h, 1, @@ -60,7 +128,7 @@ namespace { } }; - const block_trace_v1 bt2 { + const block_trace_v1 bt2_v1 { { "0000000000000000000000000000000000000000000000000000000000000002"_h, 5, @@ -152,11 +220,11 @@ namespace { struct vslice { enum mode { read_mode, write_mode}; vslice(mode m = write_mode) : _mode(m) {} - long tellp() const { + unsigned long tellp() const { return _pos; } - void seek( long loc ) { + void seek( unsigned long loc ) { if (_mode == read_mode) { if (loc > _buffer.size()) { throw std::ios_base::failure( "read vslice unable to seek to: " + std::to_string(loc) + ", end is at: " + std::to_string(_buffer.size())); @@ -200,7 +268,7 @@ namespace { std::vector _buffer; mode _mode = write_mode; - long _pos = 0l; + unsigned long _pos = 0lu; bool _flush = false; bool _sync = false; }; @@ -236,19 +304,19 @@ BOOST_AUTO_TEST_SUITE(slice_tests) BOOST_FIXTURE_TEST_CASE(write_data_trace, test_fixture) { vslice vs; - const auto offset = append_store(bt, vs ); + const auto offset = append_store(bt_v1, vs ); BOOST_REQUIRE_EQUAL(offset,0); - const auto offset2 = append_store(bt2, vs ); + const auto offset2 = append_store(bt2_v1, vs ); BOOST_REQUIRE(offset < offset2); vs._pos = offset; const auto bt_returned = extract_store( vs ); - BOOST_REQUIRE(bt_returned == bt); + BOOST_REQUIRE(bt_returned == bt_v1); vs._pos = offset2; const auto bt_returned2 = extract_store( vs ); - BOOST_REQUIRE(bt_returned2 == bt2); + BOOST_REQUIRE(bt_returned2 == bt2_v1); } BOOST_FIXTURE_TEST_CASE(write_data_multi_trace_version, test_fixture) @@ -257,16 +325,23 @@ BOOST_AUTO_TEST_SUITE(slice_tests) const auto offset = append_store(bt_v0, vs ); BOOST_REQUIRE_EQUAL(offset,0); - const auto offset2 = append_store(bt, vs ); + const auto offset2 = append_store(bt_v1, vs ); BOOST_REQUIRE(offset < offset2); + const auto offset3 = append_store(block_trace1_v2, vs ); + BOOST_REQUIRE(offset2 < offset3); + vs._pos = offset; const auto bt_returned = extract_store( vs ); BOOST_REQUIRE(bt_returned == bt_v0); vs._pos = offset2; const auto bt_returned2 = extract_store( vs ); - BOOST_REQUIRE(bt_returned2 == bt); + BOOST_REQUIRE(bt_returned2 == bt_v1); + + vs._pos = offset3; + const auto bt_returned3 = extract_store( vs ); + BOOST_REQUIRE(bt_returned3 == block_trace1_v2); } BOOST_FIXTURE_TEST_CASE(write_metadata_trace, test_fixture) @@ -289,30 +364,30 @@ BOOST_AUTO_TEST_SUITE(slice_tests) vs._pos = offset; const auto be_returned1 = extract_store( vs ); - BOOST_REQUIRE(be_returned1.contains()); - const auto real_be_returned1 = be_returned1.get(); - const auto real_be1 = be1.get(); + BOOST_REQUIRE(std::holds_alternative(be_returned1)); + const auto real_be_returned1 = std::get(be_returned1); + const auto real_be1 = std::get(be1); BOOST_REQUIRE(real_be_returned1 == real_be1); vs._pos = offset2; const auto le_returned1 = extract_store( vs ); - BOOST_REQUIRE(le_returned1.contains()); - const auto real_le_returned1 = le_returned1.get(); - const auto real_le1 = le1.get(); + BOOST_REQUIRE(std::holds_alternative(le_returned1)); + const auto real_le_returned1 = std::get(le_returned1); + const auto real_le1 = std::get(le1); BOOST_REQUIRE(real_le_returned1 == real_le1); vs._pos = offset3; const auto be_returned2 = extract_store( vs ); - BOOST_REQUIRE(be_returned2.contains()); - const auto real_be_returned2 = be_returned2.get(); - const auto real_be2 = be2.get(); + BOOST_REQUIRE(std::holds_alternative(be_returned2)); + const auto real_be_returned2 = std::get(be_returned2); + const auto real_be2 = std::get(be2); BOOST_REQUIRE(real_be_returned2 == real_be2); vs._pos = offset4; const auto le_returned2 = extract_store( vs ); - BOOST_REQUIRE(le_returned2.contains()); - const auto real_le_returned2 = le_returned2.get(); - const auto real_le2 = le2.get(); + BOOST_REQUIRE(std::holds_alternative(le_returned2)); + const auto real_le_returned2 = std::get(le_returned2); + const auto real_le2 = std::get(le2); BOOST_REQUIRE(real_le_returned2 == real_le2); } @@ -375,9 +450,9 @@ BOOST_AUTO_TEST_SUITE(slice_tests) BOOST_REQUIRE(slice.is_open()); BOOST_REQUIRE_EQUAL(bfs::file_size(fp), 0); BOOST_REQUIRE_EQUAL(slice.tellp(), 0); - uint64_t offset = append_store(bt, slice); + uint64_t offset = append_store(bt_v1, slice); BOOST_REQUIRE_EQUAL(offset, 0); - auto data = fc::raw::pack(bt); + auto data = fc::raw::pack(bt_v1); BOOST_REQUIRE(slice.tellp() > 0); BOOST_REQUIRE_EQUAL(data.size(), slice.tellp()); BOOST_REQUIRE_EQUAL(bfs::file_size(fp), slice.tellp()); @@ -631,7 +706,7 @@ BOOST_AUTO_TEST_SUITE(slice_tests) sd.run_maintenance_tasks(14, {}); verify_directory_contents(tempdir.path(), files); - for (int reps = 0; reps < file_paths.size(); reps++) { + for (std::size_t reps = 0; reps < file_paths.size(); reps++) { // leading edge, // compresses one slice files.erase(std::get<1>(file_paths.at(reps))); @@ -686,7 +761,7 @@ BOOST_AUTO_TEST_SUITE(slice_tests) sd.run_maintenance_tasks(14, {}); verify_directory_contents(tempdir.path(), files); - for (int reps = 0; reps < file_paths.size() + 1; reps++) { + for (std::size_t reps = 0; reps < file_paths.size() + 1; reps++) { // leading edge, // compresses one slice IF its not past the end of our test, if (reps < file_paths.size()) { @@ -711,25 +786,25 @@ BOOST_AUTO_TEST_SUITE(slice_tests) BOOST_REQUIRE_EQUAL(files.size(), 0); } - BOOST_FIXTURE_TEST_CASE(store_provider_write_read, test_fixture) + BOOST_FIXTURE_TEST_CASE(store_provider_write_read_v1, test_fixture) { fc::temp_directory tempdir; test_store_provider sp(tempdir.path(), 100); - sp.append(bt); + sp.append(bt_v1); sp.append_lib(54); - sp.append(bt2); - const uint32_t bt_bn = bt.number; + sp.append(bt2_v1); + const uint32_t bt_bn = bt_v1.number; bool found_block = false; bool lib_seen = false; const uint64_t first_offset = sp.scan_metadata_log_from(9, 0, [&](const metadata_log_entry& e) -> bool { - if (e.contains()) { - const auto& block = e.get(); + if (std::holds_alternative(e)) { + const auto& block = std::get(e); if (block.number == bt_bn) { BOOST_REQUIRE(!found_block); found_block = true; } - } else if (e.contains()) { - auto best_lib = e.get(); + } else if (std::holds_alternative(e)) { + auto best_lib = std::get(e); BOOST_REQUIRE(!lib_seen); BOOST_REQUIRE_EQUAL(best_lib.lib, 54); lib_seen = true; @@ -744,12 +819,12 @@ BOOST_AUTO_TEST_SUITE(slice_tests) std::vector block_offsets; lib_seen = false; uint64_t offset = sp.scan_metadata_log_from(9, 0, [&](const metadata_log_entry& e) -> bool { - if (e.contains()) { - const auto& block = e.get(); + if (std::holds_alternative(e)) { + const auto& block = std::get(e); block_nums.push_back(block.number); block_offsets.push_back(block.offset); - } else if (e.contains()) { - auto best_lib = e.get(); + } else if (std::holds_alternative(e)) { + auto best_lib = std::get(e); BOOST_REQUIRE(!lib_seen); BOOST_REQUIRE_EQUAL(best_lib.lib, 54); lib_seen = true; @@ -758,19 +833,112 @@ BOOST_AUTO_TEST_SUITE(slice_tests) }, []() {}); BOOST_REQUIRE(lib_seen); BOOST_REQUIRE_EQUAL(block_nums.size(), 2); - BOOST_REQUIRE_EQUAL(block_nums[0], bt.number); - BOOST_REQUIRE_EQUAL(block_nums[1], bt2.number); + BOOST_REQUIRE_EQUAL(block_nums[0], bt_v1.number); + BOOST_REQUIRE_EQUAL(block_nums[1], bt2_v1.number); BOOST_REQUIRE_EQUAL(block_offsets.size(), 2); BOOST_REQUIRE(block_offsets[0] < block_offsets[1]); BOOST_REQUIRE(first_offset < offset); std::optional bt_data = sp.read_data_log(block_nums[0], block_offsets[0]); + BOOST_REQUIRE_EQUAL(std::get(*bt_data), bt_v1); + + bt_data = sp.read_data_log(block_nums[1], block_offsets[1]); BOOST_REQUIRE(bt_data); - BOOST_REQUIRE_EQUAL(*bt_data, bt); + auto v = std::variant(*bt_data); + BOOST_REQUIRE_EQUAL(std::get(v), bt2_v1); + + block_nums.clear(); + block_offsets.clear(); + lib_seen = false; + int counter = 0; + try { + offset = sp.scan_metadata_log_from(9, 0, [&](const metadata_log_entry& e) -> bool { + if (std::holds_alternative(e)) { + const auto& block = std::get(e); + block_nums.push_back(block.number); + block_offsets.push_back(block.offset); + } else if (std::holds_alternative(e)) { + auto best_lib = std::get(e); + BOOST_REQUIRE(!lib_seen); + BOOST_REQUIRE_EQUAL(best_lib.lib, 54); + lib_seen = true; + } + return true; + }, [&counter]() { + if( ++counter == 3 ) { + throw yield_exception(""); + } + }); + BOOST_FAIL("Should not have completed scan"); + } catch (const yield_exception& ex) { + } + BOOST_REQUIRE(lib_seen); + BOOST_REQUIRE_EQUAL(block_nums.size(), 1); + BOOST_REQUIRE_EQUAL(block_nums[0], bt_v1.number); + BOOST_REQUIRE_EQUAL(block_offsets.size(), 1); + BOOST_REQUIRE(first_offset < offset); + } + + BOOST_FIXTURE_TEST_CASE(store_provider_write_read_v2, test_fixture) + { + fc::temp_directory tempdir; + test_store_provider sp(tempdir.path(), 100); + sp.append(block_trace1_v2); + sp.append_lib(54); + sp.append(block_trace2_v2); + const uint32_t bt_bn = block_trace1_v2.number; + bool found_block = false; + bool lib_seen = false; + const uint64_t first_offset = sp.scan_metadata_log_from(9, 0, [&](const metadata_log_entry& e) -> bool { + if (std::holds_alternative(e)) { + const auto& block = std::get(e); + if (block.number == bt_bn) { + BOOST_REQUIRE(!found_block); + found_block = true; + } + } else if (std::holds_alternative(e)) { + auto best_lib = std::get(e); + BOOST_REQUIRE(!lib_seen); + BOOST_REQUIRE_EQUAL(best_lib.lib, 54); + lib_seen = true; + return false; + } + return true; + }, []() {}); + BOOST_REQUIRE(found_block); + BOOST_REQUIRE(lib_seen); + + std::vector block_nums; + std::vector block_offsets; + lib_seen = false; + uint64_t offset = sp.scan_metadata_log_from(9, 0, [&](const metadata_log_entry& e) -> bool { + if (std::holds_alternative(e)) { + const auto& block = std::get(e); + block_nums.push_back(block.number); + block_offsets.push_back(block.offset); + } else if (std::holds_alternative(e)) { + auto best_lib = std::get(e); + BOOST_REQUIRE(!lib_seen); + BOOST_REQUIRE_EQUAL(best_lib.lib, 54); + lib_seen = true; + } + return true; + }, []() {}); + BOOST_REQUIRE(lib_seen); + BOOST_REQUIRE_EQUAL(block_nums.size(), 2); + BOOST_REQUIRE_EQUAL(block_nums[0], block_trace1_v2.number); + BOOST_REQUIRE_EQUAL(block_nums[1], block_trace2_v2.number); + BOOST_REQUIRE_EQUAL(block_offsets.size(), 2); + BOOST_REQUIRE(block_offsets[0] < block_offsets[1]); + BOOST_REQUIRE(first_offset < offset); + + std::optional bt_data = sp.read_data_log(block_nums[0], block_offsets[0]); + BOOST_REQUIRE_EQUAL(std::get(*bt_data), block_trace1_v2); bt_data = sp.read_data_log(block_nums[1], block_offsets[1]); BOOST_REQUIRE(bt_data); - BOOST_REQUIRE_EQUAL(*bt_data, bt2); + auto v = data_log_entry(*bt_data); + BOOST_REQUIRE_EQUAL(std::get(v), block_trace2_v2); block_nums.clear(); block_offsets.clear(); @@ -778,12 +946,12 @@ BOOST_AUTO_TEST_SUITE(slice_tests) int counter = 0; try { offset = sp.scan_metadata_log_from(9, 0, [&](const metadata_log_entry& e) -> bool { - if (e.contains()) { - const auto& block = e.get(); + if (std::holds_alternative(e)) { + const auto& block = std::get(e); block_nums.push_back(block.number); block_offsets.push_back(block.offset); - } else if (e.contains()) { - auto best_lib = e.get(); + } else if (std::holds_alternative(e)) { + auto best_lib = std::get(e); BOOST_REQUIRE(!lib_seen); BOOST_REQUIRE_EQUAL(best_lib.lib, 54); lib_seen = true; @@ -799,18 +967,67 @@ BOOST_AUTO_TEST_SUITE(slice_tests) } BOOST_REQUIRE(lib_seen); BOOST_REQUIRE_EQUAL(block_nums.size(), 1); - BOOST_REQUIRE_EQUAL(block_nums[0], bt.number); + BOOST_REQUIRE_EQUAL(block_nums[0], block_trace1_v2.number); BOOST_REQUIRE_EQUAL(block_offsets.size(), 1); BOOST_REQUIRE(first_offset < offset); } - BOOST_FIXTURE_TEST_CASE(test_get_block, test_fixture) + BOOST_FIXTURE_TEST_CASE(test_get_block_v1, test_fixture) + { + fc::temp_directory tempdir; + store_provider sp(tempdir.path(), 100, std::optional(), std::optional(), 0); + sp.append(bt_v1); + sp.append_lib(1); + sp.append(bt2_v1); + int count = 0; + get_block_t block1 = sp.get_block(1, [&count]() { + if (++count >= 3) { + throw yield_exception(""); + } + }); + BOOST_REQUIRE(block1); + BOOST_REQUIRE(std::get<1>(*block1)); + const auto block1_bt = std::get<0>(*block1); + BOOST_REQUIRE_EQUAL(std::get(block1_bt), bt_v1); + + count = 0; + get_block_t block2 = sp.get_block(5, [&count]() { + if (++count >= 4) { + throw yield_exception(""); + } + }); + BOOST_REQUIRE(block2); + BOOST_REQUIRE(!std::get<1>(*block2)); + const auto block2_bt = std::get<0>(*block2); + BOOST_REQUIRE_EQUAL(std::get(block2_bt), bt2_v1); + + count = 0; + try { + sp.get_block(5,[&count]() { + if (++count >= 3) { + throw yield_exception(""); + } + }); + BOOST_FAIL("Should not have completed scan"); + } catch (const yield_exception& ex) { + } + + count = 0; + block2 = sp.get_block(2,[&count]() { + if (++count >= 4) { + throw yield_exception(""); + } + }); + BOOST_REQUIRE(!block2); + } + + BOOST_FIXTURE_TEST_CASE(test_get_block_v2, test_fixture) { fc::temp_directory tempdir; store_provider sp(tempdir.path(), 100, std::optional(), std::optional(), 0); - sp.append(bt); + sp.append(block_trace1_v2); sp.append_lib(1); - sp.append(bt2); + sp.append(block_trace2_v2); int count = 0; get_block_t block1 = sp.get_block(1, [&count]() { if (++count >= 3) { @@ -820,7 +1037,7 @@ BOOST_AUTO_TEST_SUITE(slice_tests) BOOST_REQUIRE(block1); BOOST_REQUIRE(std::get<1>(*block1)); const auto block1_bt = std::get<0>(*block1); - BOOST_REQUIRE_EQUAL(block1_bt, bt); + BOOST_REQUIRE_EQUAL(std::get(block1_bt), block_trace1_v2); count = 0; get_block_t block2 = sp.get_block(5, [&count]() { @@ -831,7 +1048,7 @@ BOOST_AUTO_TEST_SUITE(slice_tests) BOOST_REQUIRE(block2); BOOST_REQUIRE(!std::get<1>(*block2)); const auto block2_bt = std::get<0>(*block2); - BOOST_REQUIRE_EQUAL(block2_bt, bt2); + BOOST_REQUIRE_EQUAL(std::get(block2_bt), block_trace2_v2); count = 0; try { @@ -853,4 +1070,5 @@ BOOST_AUTO_TEST_SUITE(slice_tests) BOOST_REQUIRE(!block2); } + BOOST_AUTO_TEST_SUITE_END() diff --git a/plugins/trace_api_plugin/trace_api.swagger.yaml b/plugins/trace_api_plugin/trace_api.swagger.yaml index 2c5f5dc78e9..62fc0c4f7d1 100644 --- a/plugins/trace_api_plugin/trace_api.swagger.yaml +++ b/plugins/trace_api_plugin/trace_api.swagger.yaml @@ -44,8 +44,8 @@ paths: application/json: schema: oneOf: - - $ref: "https://eosio.github.io/schemata/v2.0/oas/BlockTraceV0.yaml" - - $ref: "https://eosio.github.io/schemata/v2.0/oas/BlockTraceV1.yaml" + - $ref: "https://eosio.github.io/schemata/v2.1/oas/BlockTraceV0.yaml" + - $ref: "https://eosio.github.io/schemata/v2.1/oas/BlockTraceV1.yaml" "400": description: Error - requested block number is invalid (not a number, larger than max int) "404": diff --git a/plugins/trace_api_plugin/trace_api_plugin.cpp b/plugins/trace_api_plugin/trace_api_plugin.cpp index 6e47994fef7..4d0072313e1 100644 --- a/plugins/trace_api_plugin/trace_api_plugin.cpp +++ b/plugins/trace_api_plugin/trace_api_plugin.cpp @@ -7,6 +7,8 @@ #include +#include + #include using namespace eosio::trace_api; @@ -72,7 +74,8 @@ namespace { :store(store) {} - void append( const block_trace_v1& trace ) { + template + void append( const BlockTrace& trace ) { store->append(trace); } @@ -114,6 +117,8 @@ struct trace_api_common_impl { trace_dir = app().data_dir() / dir_option; else trace_dir = dir_option; + if (auto resmon_plugin = app().find_plugin()) + resmon_plugin->monitor_directory(trace_dir); slice_stride = options.at("trace-slice-stride").as(); @@ -310,7 +315,7 @@ struct trace_api_plugin_impl { auto& chain = app().find_plugin()->chain(); applied_transaction_connection.emplace( - chain.applied_transaction.connect([this](std::tuple t) { + chain.applied_transaction.connect([this](std::tuple t) { emit_killer([&](){ extraction->signal_applied_transaction(std::get<0>(t), std::get<1>(t)); }); @@ -352,10 +357,10 @@ struct trace_api_plugin_impl { using chain_extraction_t = chain_extraction_impl_type>; std::shared_ptr extraction; - fc::optional applied_transaction_connection; - fc::optional block_start_connection; - fc::optional accepted_block_connection; - fc::optional irreversible_block_connection; + std::optional applied_transaction_connection; + std::optional block_start_connection; + std::optional accepted_block_connection; + std::optional irreversible_block_connection; }; trace_api_plugin::trace_api_plugin() @@ -428,4 +433,4 @@ void trace_api_rpc_plugin::handle_sighup() { fc::logger::update( logger_name, _log ); } -} \ No newline at end of file +} diff --git a/plugins/trace_api_plugin/utils/compress_cmd.cpp b/plugins/trace_api_plugin/utils/compress_cmd.cpp index 8b7075c9532..b235ec6e40c 100644 --- a/plugins/trace_api_plugin/utils/compress_cmd.cpp +++ b/plugins/trace_api_plugin/utils/compress_cmd.cpp @@ -118,10 +118,14 @@ namespace { } catch (const bpo::error& e) { std::cerr << "Error: " << e.what() << "\n\n"; print_help_text(std::cerr, vis_desc); - } catch (const std::exception& e) { - std::cerr << "Error: " << e.what() << "\n"; + } catch ( const std::bad_alloc& ) { + throw; + } catch ( const boost::interprocess::bad_alloc& ) { + throw; } catch (const fc::exception& e) { std::cerr << "Error: " << e.to_detail_string() << "\n"; + } catch (const std::exception& e) { + std::cerr << "Error: " << e.what() << "\n"; } catch (...) { std::cerr << "An Unknown Error Occurred\n"; } diff --git a/plugins/txn_test_gen_plugin/CMakeLists.txt b/plugins/txn_test_gen_plugin/CMakeLists.txt index 5d7ca913ba8..817eaa979df 100644 --- a/plugins/txn_test_gen_plugin/CMakeLists.txt +++ b/plugins/txn_test_gen_plugin/CMakeLists.txt @@ -3,7 +3,7 @@ add_library( txn_test_gen_plugin txn_test_gen_plugin.cpp ${HEADERS} ) -target_link_libraries( txn_test_gen_plugin appbase fc http_plugin chain_plugin eosio_testing ) +target_link_libraries( txn_test_gen_plugin appbase fc http_plugin chain_plugin eosio_testing_contracts ) target_include_directories( txn_test_gen_plugin PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include txn_test_gen_plugin PUBLIC ${CMAKE_SOURCE_DIR}/libraries/testing/include txn_test_gen_plugin PUBLIC ${CMAKE_BINARY_DIR}/unittests/include ) diff --git a/plugins/txn_test_gen_plugin/txn_test_gen_plugin.cpp b/plugins/txn_test_gen_plugin/txn_test_gen_plugin.cpp index 8b7aaebfb7d..ba7e058bbd6 100644 --- a/plugins/txn_test_gen_plugin/txn_test_gen_plugin.cpp +++ b/plugins/txn_test_gen_plugin/txn_test_gen_plugin.cpp @@ -97,7 +97,7 @@ struct txn_test_gen_plugin_impl { uint64_t _txcount = 0; uint16_t thread_pool_size; - fc::optional thread_pool; + std::optional thread_pool; std::shared_ptr timer; name newaccountA; name newaccountB; @@ -107,12 +107,13 @@ struct txn_test_gen_plugin_impl { chain_plugin& cp = app().get_plugin(); for (size_t i = 0; i < trxs->size(); ++i) { - cp.accept_transaction( std::make_shared(trxs->at(i)), [=](const fc::static_variant& result){ - if (result.contains()) { - next(result.get()); + cp.accept_transaction( std::make_shared(signed_transaction(trxs->at(i)), true), + [=](const std::variant& result){ + if (std::holds_alternative(result)) { + next(std::get(result)); } else { - if (result.contains() && result.get()->receipt) { - _total_us += result.get()->receipt->cpu_usage_us; + if (std::holds_alternative(result) && std::get(result)->receipt) { + _total_us += std::get(result)->receipt->cpu_usage_us; ++_txcount; } } @@ -206,7 +207,7 @@ struct txn_test_gen_plugin_impl { { action act; act.account = newaccountT; - act.name = N(create); + act.name = "create"_n; act.authorization = vector{{newaccountT,config::active_name}}; act.data = eosio_token_serializer.variant_to_binary("create", fc::json::from_string(fc::format_string("{\"issuer\":\"${issuer}\",\"maximum_supply\":\"1000000000.0000 CUR\"}}", @@ -217,7 +218,7 @@ struct txn_test_gen_plugin_impl { { action act; act.account = newaccountT; - act.name = N(issue); + act.name = "issue"_n; act.authorization = vector{{newaccountT,config::active_name}}; act.data = eosio_token_serializer.variant_to_binary("issue", fc::json::from_string(fc::format_string("{\"to\":\"${to}\",\"quantity\":\"60000.0000 CUR\",\"memo\":\"\"}", @@ -228,7 +229,7 @@ struct txn_test_gen_plugin_impl { { action act; act.account = newaccountT; - act.name = N(transfer); + act.name = "transfer"_n; act.authorization = vector{{newaccountT,config::active_name}}; act.data = eosio_token_serializer.variant_to_binary("transfer", fc::json::from_string(fc::format_string("{\"from\":\"${from}\",\"to\":\"${to}\",\"quantity\":\"20000.0000 CUR\",\"memo\":\"\"}", @@ -239,7 +240,7 @@ struct txn_test_gen_plugin_impl { { action act; act.account = newaccountT; - act.name = N(transfer); + act.name = "transfer"_n; act.authorization = vector{{newaccountT,config::active_name}}; act.data = eosio_token_serializer.variant_to_binary("transfer", fc::json::from_string(fc::format_string("{\"from\":\"${from}\",\"to\":\"${to}\",\"quantity\":\"20000.0000 CUR\",\"memo\":\"\"}", @@ -254,9 +255,16 @@ struct txn_test_gen_plugin_impl { trx.sign(txn_test_receiver_C_priv_key, chainid); trxs.emplace_back(std::move(trx)); } + } catch ( const std::bad_alloc& ) { + throw; + } catch ( const boost::interprocess::bad_alloc& ) { + throw; } catch (const fc::exception& e) { next(e.dynamic_copy_exception()); return; + } catch (const std::exception& e) { + next(fc::std_exception_wrapper::from_current_exception(e).dynamic_copy_exception()); + return; } push_transactions(std::move(trxs), next); @@ -281,7 +289,7 @@ struct txn_test_gen_plugin_impl { abi_serializer eosio_token_serializer{fc::json::from_string(contracts::eosio_token_abi().data()).as(), abi_serializer::create_yield_function( abi_serializer_max_time )}; //create the actions here act_a_to_b.account = newaccountT; - act_a_to_b.name = N(transfer); + act_a_to_b.name = "transfer"_n; act_a_to_b.authorization = vector{{newaccountA,config::active_name}}; act_a_to_b.data = eosio_token_serializer.variant_to_binary("transfer", fc::json::from_string(fc::format_string("{\"from\":\"${from}\",\"to\":\"${to}\",\"quantity\":\"1.0000 CUR\",\"memo\":\"${l}\"}", @@ -289,7 +297,7 @@ struct txn_test_gen_plugin_impl { abi_serializer::create_yield_function( abi_serializer_max_time )); act_b_to_a.account = newaccountT; - act_b_to_a.name = N(transfer); + act_b_to_a.name = "transfer"_n; act_b_to_a.authorization = vector{{newaccountB,config::active_name}}; act_b_to_a.data = eosio_token_serializer.variant_to_binary("transfer", fc::json::from_string(fc::format_string("{\"from\":\"${from}\",\"to\":\"${to}\",\"quantity\":\"1.0000 CUR\",\"memo\":\"${l}\"}", @@ -378,8 +386,14 @@ struct txn_test_gen_plugin_impl { trxs.emplace_back(std::move(trx)); } } + } catch ( const std::bad_alloc& ) { + throw; + } catch ( const boost::interprocess::bad_alloc& ) { + throw; } catch ( const fc::exception& e ) { next(e.dynamic_copy_exception()); + } catch (const std::exception& e) { + next(fc::std_exception_wrapper::from_current_exception(e).dynamic_copy_exception()); } push_transactions(std::move(trxs), next); @@ -450,7 +464,7 @@ void txn_test_gen_plugin::plugin_shutdown() { try { my->stop_generation(); } - catch(fc::exception& e) { + catch(const std::exception& e) { } } diff --git a/plugins/wallet_api_plugin/wallet_api_plugin.cpp b/plugins/wallet_api_plugin/wallet_api_plugin.cpp index 235aa515e38..597509d04bf 100644 --- a/plugins/wallet_api_plugin/wallet_api_plugin.cpp +++ b/plugins/wallet_api_plugin/wallet_api_plugin.cpp @@ -20,11 +20,10 @@ static appbase::abstract_plugin& _wallet_api_plugin = app().register_plugin()); + auto params = parse_params(body);\ + auto result = api_handle.call_name( std::move(params) ); #define INVOKE_R_R_R(api_handle, call_name, in_param0, in_param1) \ - const auto& vs = fc::json::json::from_string(body).as(); \ - auto result = api_handle.call_name(vs.at(0).as(), vs.at(1).as()); + const auto& params = parse_params(body);\ + if (params.size() != 2) { \ + EOS_THROW(chain::invalid_http_request, "Missing valid input from POST body"); \ + } \ + auto result = api_handle.call_name(params.at(0).as(), params.at(1).as()); +// chain_id_type does not have default constructor, keep it unchanged #define INVOKE_R_R_R_R(api_handle, call_name, in_param0, in_param1, in_param2) \ - const auto& vs = fc::json::json::from_string(body).as(); \ - auto result = api_handle.call_name(vs.at(0).as(), vs.at(1).as(), vs.at(2).as()); + const auto& params = parse_params(body);\ + if (params.size() != 3) { \ + EOS_THROW(chain::invalid_http_request, "Missing valid input from POST body"); \ + } \ + auto result = api_handle.call_name(params.at(0).as(), params.at(1).as(), params.at(2).as()); #define INVOKE_R_V(api_handle, call_name) \ + body = parse_params(body); \ auto result = api_handle.call_name(); #define INVOKE_V_R(api_handle, call_name, in_param) \ - api_handle.call_name(fc::json::from_string(body).as()); \ + auto params = parse_params(body);\ + api_handle.call_name( std::move(params) ); \ eosio::detail::wallet_api_plugin_empty result; #define INVOKE_V_R_R(api_handle, call_name, in_param0, in_param1) \ - const auto& vs = fc::json::json::from_string(body).as(); \ - api_handle.call_name(vs.at(0).as(), vs.at(1).as()); \ + const auto& params = parse_params(body);\ + if (params.size() != 2) { \ + EOS_THROW(chain::invalid_http_request, "Missing valid input from POST body"); \ + } \ + api_handle.call_name(params.at(0).as(), params.at(1).as()); \ eosio::detail::wallet_api_plugin_empty result; #define INVOKE_V_R_R_R(api_handle, call_name, in_param0, in_param1, in_param2) \ - const auto& vs = fc::json::json::from_string(body).as(); \ - api_handle.call_name(vs.at(0).as(), vs.at(1).as(), vs.at(2).as()); \ + const auto& params = parse_params(body);\ + if (params.size() != 3) { \ + EOS_THROW(chain::invalid_http_request, "Missing valid input from POST body"); \ + } \ + api_handle.call_name(params.at(0).as(), params.at(1).as(), params.at(2).as()); \ eosio::detail::wallet_api_plugin_empty result; #define INVOKE_V_V(api_handle, call_name) \ + body = parse_params(body); \ api_handle.call_name(); \ eosio::detail::wallet_api_plugin_empty result; - void wallet_api_plugin::plugin_startup() { ilog("starting wallet_api_plugin"); // lifetime of plugin is lifetime of application auto& wallet_mgr = app().get_plugin().get_wallet_manager(); app().get_plugin().add_api({ - CALL(wallet, wallet_mgr, set_timeout, + CALL_WITH_400(wallet, wallet_mgr, set_timeout, INVOKE_V_R(wallet_mgr, set_timeout, int64_t), 200), - CALL(wallet, wallet_mgr, sign_transaction, - INVOKE_R_R_R_R(wallet_mgr, sign_transaction, chain::signed_transaction, flat_set, chain::chain_id_type), 201), - CALL(wallet, wallet_mgr, sign_digest, + // chain::chain_id_type has an inaccessible default constructor + CALL_WITH_400(wallet, wallet_mgr, sign_transaction, + INVOKE_R_R_R_R(wallet_mgr, sign_transaction, chain::signed_transaction, chain::flat_set, chain::chain_id_type), 201), + CALL_WITH_400(wallet, wallet_mgr, sign_digest, INVOKE_R_R_R(wallet_mgr, sign_digest, chain::digest_type, public_key_type), 201), - CALL(wallet, wallet_mgr, create, + CALL_WITH_400(wallet, wallet_mgr, create, INVOKE_R_R(wallet_mgr, create, std::string), 201), - CALL(wallet, wallet_mgr, open, + CALL_WITH_400(wallet, wallet_mgr, open, INVOKE_V_R(wallet_mgr, open, std::string), 200), - CALL(wallet, wallet_mgr, lock_all, + CALL_WITH_400(wallet, wallet_mgr, lock_all, INVOKE_V_V(wallet_mgr, lock_all), 200), - CALL(wallet, wallet_mgr, lock, + CALL_WITH_400(wallet, wallet_mgr, lock, INVOKE_V_R(wallet_mgr, lock, std::string), 200), - CALL(wallet, wallet_mgr, unlock, + CALL_WITH_400(wallet, wallet_mgr, unlock, INVOKE_V_R_R(wallet_mgr, unlock, std::string, std::string), 200), - CALL(wallet, wallet_mgr, import_key, + CALL_WITH_400(wallet, wallet_mgr, import_key, INVOKE_V_R_R(wallet_mgr, import_key, std::string, std::string), 201), - CALL(wallet, wallet_mgr, remove_key, + CALL_WITH_400(wallet, wallet_mgr, remove_key, INVOKE_V_R_R_R(wallet_mgr, remove_key, std::string, std::string, std::string), 201), - CALL(wallet, wallet_mgr, create_key, + CALL_WITH_400(wallet, wallet_mgr, create_key, INVOKE_R_R_R(wallet_mgr, create_key, std::string, std::string), 201), - CALL(wallet, wallet_mgr, list_wallets, + CALL_WITH_400(wallet, wallet_mgr, list_wallets, INVOKE_R_V(wallet_mgr, list_wallets), 200), - CALL(wallet, wallet_mgr, list_keys, + CALL_WITH_400(wallet, wallet_mgr, list_keys, INVOKE_R_R_R(wallet_mgr, list_keys, std::string, std::string), 200), - CALL(wallet, wallet_mgr, get_public_keys, + CALL_WITH_400(wallet, wallet_mgr, get_public_keys, INVOKE_R_V(wallet_mgr, get_public_keys), 200) }); } diff --git a/plugins/wallet_plugin/CMakeLists.txt b/plugins/wallet_plugin/CMakeLists.txt index 27f0147c8d7..d175119811b 100644 --- a/plugins/wallet_plugin/CMakeLists.txt +++ b/plugins/wallet_plugin/CMakeLists.txt @@ -8,10 +8,6 @@ if(APPLE) find_library(localauthentication_framework LocalAuthentication) find_library(corefoundation_framework CoreFoundation) find_library(cocoa_framework Cocoa) - - if(MAS_KEYCHAIN_GROUP) - add_definitions(-DMAS_KEYCHAIN_GROUP=${MAS_KEYCHAIN_GROUP}) - endif(MAS_KEYCHAIN_GROUP) endif(APPLE) add_library( wallet_plugin @@ -25,5 +21,9 @@ add_library( wallet_plugin target_link_libraries( wallet_plugin yubihsm_static eosio_chain appbase ${security_framework} ${corefoundation_framework} ${localauthentication_framework} ${cocoa_framework}) target_include_directories( wallet_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) +if(APPLE) + target_link_libraries( wallet_plugin se-helpers ) +endif() + #sadly old cmake 2.8 support in yubihsm cmake prevents usage of target_include_directories there target_include_directories( wallet_plugin PRIVATE "${CMAKE_SOURCE_DIR}/libraries/yubihsm/lib" ) \ No newline at end of file diff --git a/plugins/wallet_plugin/include/eosio/wallet_plugin/se_wallet.hpp b/plugins/wallet_plugin/include/eosio/wallet_plugin/se_wallet.hpp index 9e33b194a37..62329d6af3d 100644 --- a/plugins/wallet_plugin/include/eosio/wallet_plugin/se_wallet.hpp +++ b/plugins/wallet_plugin/include/eosio/wallet_plugin/se_wallet.hpp @@ -32,7 +32,7 @@ class se_wallet final : public wallet_api { string create_key(string key_type) override; bool remove_key(string key) override; - fc::optional try_sign_digest(const digest_type digest, const public_key_type public_key) override; + std::optional try_sign_digest(const digest_type digest, const public_key_type public_key) override; private: std::unique_ptr my; diff --git a/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet.hpp b/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet.hpp index 92f2c0e2276..f1468cdb3ab 100644 --- a/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet.hpp +++ b/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet.hpp @@ -177,7 +177,7 @@ class soft_wallet final : public wallet_api /* Attempts to sign a digest via the given public_key */ - fc::optional try_sign_digest( const digest_type digest, const public_key_type public_key ) override; + std::optional try_sign_digest( const digest_type digest, const public_key_type public_key ) override; std::shared_ptr my; void encrypt_keys(); diff --git a/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet_api.hpp b/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet_api.hpp index bdca3ee93e1..4810898c9c1 100644 --- a/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet_api.hpp +++ b/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet_api.hpp @@ -97,7 +97,7 @@ class wallet_api /** Returns a signature given the digest and public_key, if this wallet can sign via that public key */ - virtual fc::optional try_sign_digest( const digest_type digest, const public_key_type public_key ) = 0; + virtual std::optional try_sign_digest( const digest_type digest, const public_key_type public_key ) = 0; }; }} diff --git a/plugins/wallet_plugin/include/eosio/wallet_plugin/yubihsm_wallet.hpp b/plugins/wallet_plugin/include/eosio/wallet_plugin/yubihsm_wallet.hpp index 49caa9c184b..abafbe8478c 100644 --- a/plugins/wallet_plugin/include/eosio/wallet_plugin/yubihsm_wallet.hpp +++ b/plugins/wallet_plugin/include/eosio/wallet_plugin/yubihsm_wallet.hpp @@ -32,7 +32,7 @@ class yubihsm_wallet final : public wallet_api { string create_key(string key_type) override; bool remove_key(string key) override; - fc::optional try_sign_digest(const digest_type digest, const public_key_type public_key) override; + std::optional try_sign_digest(const digest_type digest, const public_key_type public_key) override; private: std::unique_ptr my; diff --git a/plugins/wallet_plugin/se_wallet.cpp b/plugins/wallet_plugin/se_wallet.cpp index dc919554e0b..5867a89180f 100644 --- a/plugins/wallet_plugin/se_wallet.cpp +++ b/plugins/wallet_plugin/se_wallet.cpp @@ -1,11 +1,7 @@ #include #include #include - -#include - -#include -#include +#include #include @@ -13,8 +9,6 @@ namespace eosio { namespace wallet { -using namespace fc::crypto::r1; - namespace detail { static void auth_callback(int success, void* data) { @@ -23,295 +17,19 @@ static void auth_callback(int success, void* data) { } struct se_wallet_impl { - - static public_key_data get_public_key_data(SecKeyRef key) { - SecKeyRef pubkey = SecKeyCopyPublicKey(key); - - CFErrorRef error = nullptr; - CFDataRef keyrep = nullptr; - keyrep = SecKeyCopyExternalRepresentation(pubkey, &error); - - public_key_data pub_key_data; - if(!error) { - const UInt8* cfdata = CFDataGetBytePtr(keyrep); - memcpy(pub_key_data.data+1, cfdata+1, 32); - pub_key_data.data[0] = 0x02 + (cfdata[64]&1); - } - - CFRelease(keyrep); - CFRelease(pubkey); - - if(error) { - string error_string = string_for_cferror(error); - CFRelease(error); - FC_THROW_EXCEPTION(chain::wallet_exception, "Failed to get public key from Secure Enclave: ${m}", ("m", error_string)); - } - - return pub_key_data; - } - - static public_key_type get_public_key(SecKeyRef key) { - char serialized_pub_key[sizeof(public_key_data) + 1]; - serialized_pub_key[0] = 0x01; - - public_key_data pub_key_data = get_public_key_data(key); - memcpy(serialized_pub_key+1, pub_key_data.data, sizeof(pub_key_data)); - - public_key_type pub_key; - fc::datastream ds(serialized_pub_key, sizeof(serialized_pub_key)); - fc::raw::unpack(ds, pub_key); - - return pub_key; - } - - static string string_for_cferror(CFErrorRef error) { - CFStringRef errorString = CFCopyDescription(error); - char buff[CFStringGetLength(errorString) + 1]; - string ret; - if(CFStringGetCString(errorString, buff, sizeof(buff), kCFStringEncodingUTF8)) - ret = buff; - else - ret = "Unknown"; - CFRelease(errorString); - return ret; - } - -#define XSTR(A) STR(A) -#define STR(A) #A - - void populate_existing_keys() { - const void* keyAttrKeys[] = { - kSecClass, - kSecAttrKeyClass, - kSecMatchLimit, - kSecReturnRef, - kSecAttrTokenID, - kSecAttrAccessGroup - }; - const void* keyAttrValues[] = { - kSecClassKey, - kSecAttrKeyClassPrivate, - kSecMatchLimitAll, - kCFBooleanTrue, - kSecAttrTokenIDSecureEnclave, -#ifdef MAS_KEYCHAIN_GROUP - CFSTR(XSTR(MAS_KEYCHAIN_GROUP)) -#endif - }; - CFDictionaryRef keyAttrDic = CFDictionaryCreate(nullptr, keyAttrKeys, keyAttrValues, sizeof(keyAttrValues)/sizeof(keyAttrValues[0]), &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); - - CFArrayRef keyRefs = nullptr; - if(SecItemCopyMatching(keyAttrDic, (CFTypeRef*)&keyRefs) || !keyRefs) { - CFRelease(keyAttrDic); - return; - } - - CFIndex count = CFArrayGetCount(keyRefs); - for(long i = 0; i < count; ++i) { - public_key_type pub; - try { - SecKeyRef key = (SecKeyRef)CFRetain(CFArrayGetValueAtIndex(keyRefs, i)); - _keys[get_public_key(key)] = key; - } - catch(chain::wallet_exception&) {} - } - CFRelease(keyRefs); - CFRelease(keyAttrDic); - } - - public_key_type create() { - SecAccessControlRef accessControlRef = SecAccessControlCreateWithFlags(nullptr, kSecAttrAccessibleWhenUnlockedThisDeviceOnly, kSecAccessControlPrivateKeyUsage, nullptr); - - int keySizeValue = 256; - CFNumberRef keySizeNumber = CFNumberCreate(NULL, kCFNumberIntType, &keySizeValue); - - const void* keyAttrKeys[] = { - kSecAttrIsPermanent, - kSecAttrAccessControl, - kSecAttrAccessGroup - }; - const void* keyAttrValues[] = { - kCFBooleanTrue, - accessControlRef, -#ifdef MAS_KEYCHAIN_GROUP - CFSTR(XSTR(MAS_KEYCHAIN_GROUP)) -#endif - }; - CFDictionaryRef keyAttrDic = CFDictionaryCreate(NULL, keyAttrKeys, keyAttrValues, sizeof(keyAttrValues)/sizeof(keyAttrValues[0]), &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); - - const void* attrKeys[] = { - kSecAttrKeyType, - kSecAttrKeySizeInBits, - kSecAttrTokenID, - kSecPrivateKeyAttrs - }; - const void* atrrValues[] = { - kSecAttrKeyTypeECSECPrimeRandom, - keySizeNumber, - kSecAttrTokenIDSecureEnclave, - keyAttrDic - }; - CFDictionaryRef attributesDic = CFDictionaryCreate(NULL, attrKeys, atrrValues, sizeof(attrKeys)/sizeof(attrKeys[0]), &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); - - CFErrorRef error = NULL; - SecKeyRef privateKey = SecKeyCreateRandomKey(attributesDic, &error); - string error_string; - if(error) { - error_string = string_for_cferror(error); - CFRelease(error); - } - - CFRelease(attributesDic); - CFRelease(keyAttrDic); - CFRelease(keySizeNumber); - CFRelease(accessControlRef); - - if(error_string.size()) - FC_THROW_EXCEPTION(chain::wallet_exception, "Failed to create key in Secure Enclave: ${m}", ("m", error_string)); - - public_key_type pub; - try { - pub = get_public_key(privateKey); - } - catch(chain::wallet_exception&) { - //possibly we should delete the key here? - CFRelease(privateKey); - throw; - } - _keys[pub] = privateKey; - return pub; - } - - fc::optional try_sign_digest(const digest_type d, const public_key_type public_key) { - auto it = _keys.find(public_key); - if(it == _keys.end()) - return fc::optional{}; - - fc::ecdsa_sig sig = ECDSA_SIG_new(); - CFErrorRef error = nullptr; - - CFDataRef digestData = CFDataCreateWithBytesNoCopy(nullptr, (UInt8*)d.data(), d.data_size(), kCFAllocatorNull); - CFDataRef signature = SecKeyCreateSignature(it->second, kSecKeyAlgorithmECDSASignatureDigestX962SHA256, digestData, &error); - if(error) { - string error_string = string_for_cferror(error); - CFRelease(error); - CFRelease(digestData); - FC_THROW_EXCEPTION(chain::wallet_exception, "Failed to sign digest in Secure Enclave: ${m}", ("m", error_string)); - } - - const UInt8* der_bytes = CFDataGetBytePtr(signature); - long derSize = CFDataGetLength(signature); - d2i_ECDSA_SIG(&sig.obj, &der_bytes, derSize); - - public_key_data kd; - compact_signature compact_sig; - try { - kd = get_public_key_data(it->second); - compact_sig = signature_from_ecdsa(key, kd, sig, d); - } catch(chain::wallet_exception&) { - CFRelease(signature); - CFRelease(digestData); - throw; - } - - CFRelease(signature); - CFRelease(digestData); - - char serialized_signature[sizeof(compact_sig) + 1]; - serialized_signature[0] = 0x01; - memcpy(serialized_signature+1, compact_sig.data, sizeof(compact_sig)); - - signature_type final_signature; - fc::datastream ds(serialized_signature, sizeof(serialized_signature)); - fc::raw::unpack(ds, final_signature); - return final_signature; - } - - bool remove_key(string public_key) { - auto it = _keys.find(public_key_type{public_key}); - if(it == _keys.end()) - FC_THROW_EXCEPTION(chain::wallet_exception, "Given key to delete not found in Secure Enclave wallet"); - - promise prom; - future fut = prom.get_future(); - macos_user_auth(auth_callback, &prom, CFSTR("remove a key from your EOSIO wallet")); - if(!fut.get()) - FC_THROW_EXCEPTION(chain::wallet_invalid_password_exception, "Local user authentication failed"); - - CFDictionaryRef deleteDic = CFDictionaryCreate(nullptr, (const void**)&kSecValueRef, (const void**)&it->second, 1, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); - - OSStatus ret = SecItemDelete(deleteDic); - CFRelease(deleteDic); - - if(ret) - FC_THROW_EXCEPTION(chain::wallet_exception, "Failed to getremove key from Secure Enclave"); - - CFRelease(it->second); - _keys.erase(it); - - return true; - } - - ~se_wallet_impl() { - for(auto& k : _keys) - CFRelease(k.second); - } - - map _keys; - fc::ec_key key = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1); bool locked = true; }; -static void check_signed() { - OSStatus is_valid{0}; - pid_t pid = getpid(); - SecCodeRef code = nullptr; - CFNumberRef pidnumber = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pid); - CFDictionaryRef piddict = CFDictionaryCreate(kCFAllocatorDefault, (const void**)&kSecGuestAttributePid, (const void**)&pidnumber, 1, nullptr, nullptr); - if(!SecCodeCopyGuestWithAttributes(nullptr, piddict, kSecCSDefaultFlags, &code)) { - is_valid = SecCodeCheckValidity(code, kSecCSDefaultFlags, 0); - CFRelease(code); - } - CFRelease(piddict); - CFRelease(pidnumber); - - if(is_valid != errSecSuccess) { - wlog("Application does not have a valid signature; Secure Enclave support disabled"); - EOS_THROW(secure_enclave_exception, ""); - } -} - } se_wallet::se_wallet() : my(new detail::se_wallet_impl()) { - detail::check_signed(); - - //How to figure out of SE is available?! - char model[256]; - size_t model_size = sizeof(model); - if(sysctlbyname("hw.model", model, &model_size, nullptr, 0) == 0) { - if(strncmp(model, "iMacPro", strlen("iMacPro")) == 0) { - my->populate_existing_keys(); - return; - } - unsigned int major, minor; - if(sscanf(model, "MacBookPro%u,%u", &major, &minor) == 2) { - if((major >= 15) || (major >= 13 && minor >= 2)) { - my->populate_existing_keys(); - return; - } - } - if(sscanf(model, "Macmini%u", &major) == 1 && major >= 8) { - my->populate_existing_keys(); - return; - } - if(sscanf(model, "MacBookAir%u", &major) == 1 && major >= 8) { - my->populate_existing_keys(); - return; - } + if(!secure_enclave::application_signed()) { + wlog("Application does not have a valid signature; Secure Enclave support disabled"); + EOS_THROW(secure_enclave_exception, ""); } - EOS_THROW(secure_enclave_exception, "Secure Enclave not supported on this hardware"); + if(!secure_enclave::hardware_supports_secure_enclave()) + EOS_THROW(secure_enclave_exception, "Secure Enclave not supported on this hardware"); } se_wallet::~se_wallet() { @@ -348,9 +66,10 @@ map se_wallet::list_keys() { FC_THROW_EXCEPTION(chain::wallet_exception, "Getting the private keys from the Secure Enclave wallet is impossible"); } flat_set se_wallet::list_public_keys() { - flat_set keys; - boost::copy(my->_keys | boost::adaptors::map_keys, std::inserter(keys, keys.end())); - return keys; + flat_set ret; + for(const secure_enclave::secure_enclave_key& sekey : secure_enclave::get_all_keys()) + ret.emplace(sekey.public_key()); + return ret; } bool se_wallet::import_key(string wif_key) { @@ -359,16 +78,31 @@ bool se_wallet::import_key(string wif_key) { string se_wallet::create_key(string key_type) { EOS_ASSERT(key_type.empty() || key_type == "R1", chain::unsupported_key_type_exception, "Secure Enclave wallet only supports R1 keys"); - return my->create().to_string(); + return secure_enclave::create_key().public_key().to_string(); } bool se_wallet::remove_key(string key) { EOS_ASSERT(!is_locked(), wallet_locked_exception, "You can not remove a key from a locked wallet"); - return my->remove_key(key); + + auto se_keys = secure_enclave::get_all_keys(); + + for(auto it = se_keys.begin(); it != se_keys.end(); ++it) + if(it->public_key().to_string() == key) { + eosio::secure_enclave::delete_key(std::move(se_keys.extract(it).value())); + return true; + } + + FC_THROW_EXCEPTION(chain::wallet_exception, "Given key to delete not found in Secure Enclave wallet"); } -fc::optional se_wallet::try_sign_digest(const digest_type digest, const public_key_type public_key) { - return my->try_sign_digest(digest, public_key); +std::optional se_wallet::try_sign_digest(const digest_type digest, const public_key_type public_key) { + auto se_keys = secure_enclave::get_all_keys(); + + for(auto it = se_keys.begin(); it != se_keys.end(); ++it) + if(it->public_key() == public_key) + return it->sign(digest); + + return std::optional(); } }} diff --git a/plugins/wallet_plugin/wallet.cpp b/plugins/wallet_plugin/wallet.cpp index c71b52993e5..a4859a584ec 100644 --- a/plugins/wallet_plugin/wallet.cpp +++ b/plugins/wallet_plugin/wallet.cpp @@ -15,7 +15,6 @@ #include #include #include -#include #include #include @@ -116,18 +115,18 @@ class soft_wallet_impl string get_wallet_filename() const { return _wallet_filename; } - fc::optional try_get_private_key(const public_key_type& id)const + std::optional try_get_private_key(const public_key_type& id)const { auto it = _keys.find(id); if( it != _keys.end() ) return it->second; - return fc::optional(); + return std::optional(); } - fc::optional try_sign_digest( const digest_type digest, const public_key_type public_key ) { + std::optional try_sign_digest( const digest_type digest, const public_key_type public_key ) { auto it = _keys.find(public_key); if( it == _keys.end() ) - return fc::optional{}; + return std::optional(); return it->second.sign(digest); } @@ -397,7 +396,7 @@ private_key_type soft_wallet::get_private_key( public_key_type pubkey )const return my->get_private_key( pubkey ); } -fc::optional soft_wallet::try_sign_digest( const digest_type digest, const public_key_type public_key ) { +std::optional soft_wallet::try_sign_digest( const digest_type digest, const public_key_type public_key ) { return my->try_sign_digest(digest, public_key); } diff --git a/plugins/wallet_plugin/wallet_manager.cpp b/plugins/wallet_plugin/wallet_manager.cpp index 0b3bbc6c696..a8e0ba8f784 100644 --- a/plugins/wallet_plugin/wallet_manager.cpp +++ b/plugins/wallet_plugin/wallet_manager.cpp @@ -26,7 +26,7 @@ wallet_manager::wallet_manager() { #ifdef __APPLE__ try { wallets.emplace("SecureEnclave", std::make_unique()); - } catch(fc::exception& ) {} + } catch(const std::exception& ) {} #endif } @@ -233,7 +233,7 @@ wallet_manager::sign_transaction(const chain::signed_transaction& txn, const fla bool found = false; for (const auto& i : wallets) { if (!i.second->is_locked()) { - fc::optional sig = i.second->try_sign_digest(stxn.sig_digest(id, stxn.context_free_data), pk); + std::optional sig = i.second->try_sign_digest(stxn.sig_digest(id, stxn.context_free_data), pk); if (sig) { stxn.signatures.push_back(*sig); found = true; @@ -256,7 +256,7 @@ wallet_manager::sign_digest(const chain::digest_type& digest, const public_key_t try { for (const auto& i : wallets) { if (!i.second->is_locked()) { - fc::optional sig = i.second->try_sign_digest(digest, key); + std::optional sig = i.second->try_sign_digest(digest, key); if (sig) return *sig; } diff --git a/plugins/wallet_plugin/wallet_plugin.cpp b/plugins/wallet_plugin/wallet_plugin.cpp index 1b1d6aa602b..519e8bbb922 100644 --- a/plugins/wallet_plugin/wallet_plugin.cpp +++ b/plugins/wallet_plugin/wallet_plugin.cpp @@ -42,9 +42,10 @@ void wallet_plugin::plugin_initialize(const variables_map& options) { if (options.count("wallet-dir")) { auto dir = options.at("wallet-dir").as(); if (dir.is_relative()) - wallet_manager_ptr->set_dir(app().data_dir() / dir); - else - wallet_manager_ptr->set_dir(dir); + dir = app().data_dir() / dir; + if( !bfs::exists(dir) ) + bfs::create_directories(dir); + wallet_manager_ptr->set_dir(dir); } if (options.count("unlock-timeout")) { auto timeout = options.at("unlock-timeout").as(); diff --git a/plugins/wallet_plugin/yubihsm_wallet.cpp b/plugins/wallet_plugin/yubihsm_wallet.cpp index 0ff2d83116f..d2de96ae6d1 100644 --- a/plugins/wallet_plugin/yubihsm_wallet.cpp +++ b/plugins/wallet_plugin/yubihsm_wallet.cpp @@ -133,10 +133,10 @@ struct yubihsm_wallet_impl { }); } - fc::optional try_sign_digest(const digest_type d, const public_key_type public_key) { + std::optional try_sign_digest(const digest_type d, const public_key_type public_key) { auto it = _keys.find(public_key); if(it == _keys.end()) - return fc::optional{}; + return std::optional(); size_t der_sig_sz = 128; uint8_t der_sig[der_sig_sz]; @@ -261,7 +261,7 @@ bool yubihsm_wallet::remove_key(string key) { return true; } -fc::optional yubihsm_wallet::try_sign_digest(const digest_type digest, const public_key_type public_key) { +std::optional yubihsm_wallet::try_sign_digest(const digest_type digest, const public_key_type public_key) { return my->try_sign_digest(digest, public_key); } diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index a40ef3b423c..5eb775072b7 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -3,3 +3,4 @@ add_subdirectory( cleos ) add_subdirectory( keosd ) add_subdirectory( eosio-launcher ) add_subdirectory( eosio-blocklog ) +add_subdirectory( nodeos-sectl ) diff --git a/programs/cleos/CLI11.hpp b/programs/cleos/CLI11.hpp index 8f958076a81..68244d3864d 100644 --- a/programs/cleos/CLI11.hpp +++ b/programs/cleos/CLI11.hpp @@ -1,11 +1,11 @@ #pragma once -// CLI11: Version 1.9.0 +// CLI11: Version 1.9.1 // Originally designed by Henry Schreiner // https://github.com/CLIUtils/CLI11 // // This is a standalone header file generated by MakeSingleHeader.py in CLI11/scripts -// from: v1.9.0 +// from: v1.9.1 // // From LICENSE: // @@ -62,18 +62,18 @@ #include -// Verbatim copy from CLI/Version.hpp: +// Verbatim copy from Version.hpp: #define CLI11_VERSION_MAJOR 1 #define CLI11_VERSION_MINOR 9 -#define CLI11_VERSION_PATCH 0 -#define CLI11_VERSION "1.9.0" +#define CLI11_VERSION_PATCH 1 +#define CLI11_VERSION "1.9.1" -// Verbatim copy from CLI/Macros.hpp: +// Verbatim copy from Macros.hpp: // The following version macro is very similar to the one in PyBind11 @@ -112,7 +112,7 @@ -// Verbatim copy from CLI/Validators.hpp: +// Verbatim copy from Validators.hpp: // C standard library @@ -125,7 +125,14 @@ #else #include #if defined __cpp_lib_filesystem && __cpp_lib_filesystem >= 201703 +#if defined _GLIBCXX_RELEASE && _GLIBCXX_RELEASE >= 9 #define CLI11_HAS_FILESYSTEM 1 +#elif defined(__GLIBCXX__) +// if we are using gcc and Version <9 default to no filesystem +#define CLI11_HAS_FILESYSTEM 0 +#else +#define CLI11_HAS_FILESYSTEM 1 +#endif #else #define CLI11_HAS_FILESYSTEM 0 #endif @@ -134,7 +141,7 @@ #endif #if defined CLI11_HAS_FILESYSTEM && CLI11_HAS_FILESYSTEM > 0 -#include // NOLINT(build/include) +#include // NOLINT(build/include) #else #include #include @@ -142,15 +149,15 @@ -// From CLI/Version.hpp: +// From Version.hpp: -// From CLI/Macros.hpp: +// From Macros.hpp: -// From CLI/StringTools.hpp: +// From StringTools.hpp: namespace CLI { @@ -165,7 +172,7 @@ std::ostream &operator<<(std::ostream &in, const T &item) { return in << static_cast::type>(item); } -} // namespace enums +} // namespace enums /// Export to CLI namespace using enums::operator<<; @@ -179,9 +186,9 @@ constexpr int expected_max_vector_size{1 << 29}; inline std::vector split(const std::string &s, char delim) { std::vector elems; // Check to see if empty string, give consistent result - if(s.empty()) + if(s.empty()) { elems.emplace_back(); - else { + } else { std::stringstream ss; ss.str(s); std::string item; @@ -401,8 +408,9 @@ inline std::ptrdiff_t find_member(std::string name, it = std::find_if(std::begin(names), std::end(names), [&name](std::string local_name) { return detail::remove_underscore(local_name) == name; }); - } else + } else { it = std::find(std::begin(names), std::end(names), name); + } return (it != std::end(names)) ? (it - std::begin(names)) : (-1); } @@ -434,7 +442,7 @@ inline std::vector split_up(std::string str, char delimiter = '\0') if(delims.find_first_of(str[0]) != std::string::npos) { keyChar = str[0]; auto end = str.find_first_of(keyChar, 1); - while((end != std::string::npos) && (str[end - 1] == '\\')) { // deal with escaped quotes + while((end != std::string::npos) && (str[end - 1] == '\\')) { // deal with escaped quotes end = str.find_first_of(keyChar, end + 1); embeddedQuote = true; } @@ -492,7 +500,7 @@ inline std::size_t escape_detect(std::string &str, std::size_t offset) { auto astart = str.find_last_of("-/ \"\'`", offset - 1); if(astart != std::string::npos) { if(str[astart] == ((str[offset] == '=') ? '-' : '/')) - str[offset] = ' '; // interpret this as a space so the split_up works properly + str[offset] = ' '; // interpret this as a space so the split_up works properly } } return offset + 1; @@ -510,11 +518,11 @@ inline std::string &add_quotes_if_needed(std::string &str) { return str; } -} // namespace detail +} // namespace detail -} // namespace CLI +} // namespace CLI -// From CLI/Error.hpp: +// From Error.hpp: namespace CLI { @@ -747,11 +755,11 @@ class RequiredError : public ParseError { class ArgumentMismatch : public ParseError { CLI11_ERROR_DEF(ParseError, ArgumentMismatch) CLI11_ERROR_SIMPLE(ArgumentMismatch) - ArgumentMismatch(std::string name, int expected, std::size_t recieved) + ArgumentMismatch(std::string name, int expected, std::size_t received) : ArgumentMismatch(expected > 0 ? ("Expected exactly " + std::to_string(expected) + " arguments to " + name + - ", got " + std::to_string(recieved)) + ", got " + std::to_string(received)) : ("Expected at least " + std::to_string(-expected) + " arguments to " + name + - ", got " + std::to_string(recieved)), + ", got " + std::to_string(received)), ExitCodes::ArgumentMismatch) {} static ArgumentMismatch AtLeast(std::string name, int num, std::size_t received) { @@ -838,9 +846,9 @@ class OptionNotFound : public Error { /// @} -} // namespace CLI +} // namespace CLI -// From CLI/TypeTools.hpp: +// From TypeTools.hpp: namespace CLI { @@ -854,7 +862,7 @@ enum class enabler {}; /// An instance to use in EnableIf constexpr enabler dummy = {}; -} // namespace detail +} // namespace detail /// A copy of enable_if_t from C++14, compatible with C++11. /// @@ -1049,15 +1057,24 @@ template class is_tuple_like { }; /// Convert an object to a string (directly forward if this can become a string) -template ::value, detail::enabler> = detail::dummy> +template ::value, detail::enabler> = detail::dummy> auto to_string(T &&value) -> decltype(std::forward(value)) { return std::forward(value); } +/// Construct a string from the object +template ::value && !std::is_convertible::value, + detail::enabler> = detail::dummy> +std::string to_string(const T &value) { + return std::string(value); +} + /// Convert an object to a string (streaming must be supported for that type) template ::value && is_ostreamable::value, detail::enabler> = - detail::dummy> + enable_if_t::value && !std::is_constructible::value && + is_ostreamable::value, + detail::enabler> = detail::dummy> std::string to_string(T &&value) { std::stringstream stream; stream << value; @@ -1379,7 +1396,7 @@ inline std::string type_name() { // Lexical cast /// Convert a flag into an integer value typically binary flags -inline int64_t to_flag_value(std::string val) { +inline std::int64_t to_flag_value(std::string val) { static const std::string trueString("true"); static const std::string falseString("false"); if(val == trueString) { @@ -1389,10 +1406,10 @@ inline int64_t to_flag_value(std::string val) { return -1; } val = detail::to_lower(val); - int64_t ret; + std::int64_t ret; if(val.size() == 1) { if(val[0] >= '1' && val[0] <= '9') { - return (static_cast(val[0]) - '0'); + return (static_cast(val[0]) - '0'); } switch(val[0]) { case '0': @@ -1442,7 +1459,7 @@ template ::value == object_category::unsigned_integral, detail::enabler> = detail::dummy> bool lexical_cast(const std::string &input, T &output) { if(!input.empty() && input.front() == '-') - return false; // std::stoull happily converts negative values to junk without any errors. + return false; // std::stoull happily converts negative values to junk without any errors. try { std::size_t n = 0; @@ -1622,7 +1639,7 @@ bool lexical_assign(const std::string &input, T &output) { XC val{}; bool parse_result = input.empty() ? true : lexical_cast(input, val); if(parse_result) { - output = T(val); // use () form of constructor to allow some implicit conversions + output = T(val); // use () form of constructor to allow some implicit conversions } return parse_result; } @@ -1803,7 +1820,7 @@ bool lexical_conversion(const std::vector &strings, T &output) { template ::value && std::is_unsigned::value, detail::enabler> = detail::dummy> void sum_flag_vector(const std::vector &flags, T &output) { - int64_t count{0}; + std::int64_t count{0}; for(auto &flag : flags) { count += detail::to_flag_value(flag); } @@ -1818,17 +1835,17 @@ void sum_flag_vector(const std::vector &flags, T &output) { template ::value && std::is_signed::value, detail::enabler> = detail::dummy> void sum_flag_vector(const std::vector &flags, T &output) { - int64_t count{0}; + std::int64_t count{0}; for(auto &flag : flags) { count += detail::to_flag_value(flag); } output = static_cast(count); } -} // namespace detail -} // namespace CLI +} // namespace detail +} // namespace CLI -// From CLI/Split.hpp: +// From Split.hpp: namespace CLI { namespace detail { @@ -1950,10 +1967,10 @@ get_names(const std::vector &input) { short_names, long_names, pos_name); } -} // namespace detail -} // namespace CLI +} // namespace detail +} // namespace CLI -// From CLI/ConfigFwd.hpp: +// From ConfigFwd.hpp: namespace CLI { @@ -2068,9 +2085,9 @@ class ConfigTOML : public ConfigINI { valueDelimiter = '='; } }; -} // namespace CLI +} // namespace CLI -// From CLI/Validators.hpp: +// From Validators.hpp: namespace CLI { @@ -2129,14 +2146,14 @@ class Validator { } } return retstring; - }; + } /// This is the required operator for a Validator - provided to help /// users (CLI11 uses the member `func` directly) std::string operator()(const std::string &str) const { std::string value = str; return (active_) ? func_(value) : std::string{}; - }; + } /// Specify the type string Validator &description(std::string validator_desc) { @@ -2190,13 +2207,13 @@ class Validator { Validator &application_index(int app_index) { application_index_ = app_index; return *this; - }; + } /// Specify the application index of a validator Validator application_index(int app_index) const { Validator newval(*this); newval.application_index_ = app_index; return newval; - }; + } /// Get the current value of the application index int get_application_index() const { return application_index_; } /// Get a boolean if the validator is active @@ -2292,7 +2309,7 @@ class Validator { return std::string(1, '(') + f1 + ')' + merger + '(' + f2 + ')'; }; } -}; // namespace CLI +}; // namespace CLI /// Class wrapping some of the accessors of Validator class CustomValidator : public Validator { @@ -2304,7 +2321,7 @@ class CustomValidator : public Validator { namespace detail { /// CLI enumeration of different file types -enum class path_type { nonexistant, file, directory }; +enum class path_type { nonexistent, file, directory }; #if defined CLI11_HAS_FILESYSTEM && CLI11_HAS_FILESYSTEM > 0 /// get the type of the path from a file name @@ -2312,12 +2329,12 @@ inline path_type check_path(const char *file) noexcept { std::error_code ec; auto stat = std::filesystem::status(file, ec); if(ec) { - return path_type::nonexistant; + return path_type::nonexistent; } switch(stat.type()) { case std::filesystem::file_type::none: case std::filesystem::file_type::not_found: - return path_type::nonexistant; + return path_type::nonexistent; case std::filesystem::file_type::directory: return path_type::directory; case std::filesystem::file_type::symlink: @@ -2345,7 +2362,7 @@ inline path_type check_path(const char *file) noexcept { return ((buffer.st_mode & S_IFDIR) != 0) ? path_type::directory : path_type::file; } #endif - return path_type::nonexistant; + return path_type::nonexistent; } #endif /// Check for an existing file (returns error message if check fails) @@ -2354,7 +2371,7 @@ class ExistingFileValidator : public Validator { ExistingFileValidator() : Validator("FILE") { func_ = [](std::string &filename) { auto path_result = check_path(filename.c_str()); - if(path_result == path_type::nonexistant) { + if(path_result == path_type::nonexistent) { return "File does not exist: " + filename; } if(path_result == path_type::directory) { @@ -2371,7 +2388,7 @@ class ExistingDirectoryValidator : public Validator { ExistingDirectoryValidator() : Validator("DIR") { func_ = [](std::string &filename) { auto path_result = check_path(filename.c_str()); - if(path_result == path_type::nonexistant) { + if(path_result == path_type::nonexistent) { return "Directory does not exist: " + filename; } if(path_result == path_type::file) { @@ -2388,7 +2405,7 @@ class ExistingPathValidator : public Validator { ExistingPathValidator() : Validator("PATH(existing)") { func_ = [](std::string &filename) { auto path_result = check_path(filename.c_str()); - if(path_result == path_type::nonexistant) { + if(path_result == path_type::nonexistent) { return "Path does not exist: " + filename; } return std::string(); @@ -2402,7 +2419,7 @@ class NonexistentPathValidator : public Validator { NonexistentPathValidator() : Validator("PATH(non-existing)") { func_ = [](std::string &filename) { auto path_result = check_path(filename.c_str()); - if(path_result != path_type::nonexistant) { + if(path_result != path_type::nonexistent) { return "Path already exists: " + filename; } return std::string(); @@ -2481,7 +2498,7 @@ class Number : public Validator { } }; -} // namespace detail +} // namespace detail // Static is not needed here, because global const implies static. @@ -2583,7 +2600,7 @@ typename std::remove_reference::type &smart_deref(T &value) { /// Generate a string representation of a set template std::string generate_set(const T &set) { using element_t = typename detail::element_type::type; - using iteration_type_t = typename detail::pair_adaptor::value_type; // the type of the object pair + using iteration_type_t = typename detail::pair_adaptor::value_type; // the type of the object pair std::string out(1, '{'); out.append(detail::join( detail::smart_deref(set), @@ -2596,7 +2613,7 @@ template std::string generate_set(const T &set) { /// Generate a string representation of a map template std::string generate_map(const T &map, bool key_only = false) { using element_t = typename detail::element_type::type; - using iteration_type_t = typename detail::pair_adaptor::value_type; // the type of the object pair + using iteration_type_t = typename detail::pair_adaptor::value_type; // the type of the object pair std::string out(1, '{'); out.append(detail::join( detail::smart_deref(map), @@ -2707,7 +2724,7 @@ typename std::enable_if::value, bool>::type checked_mu return true; } -} // namespace detail +} // namespace detail /// Verify items are in a set class IsMember : public Validator { public: @@ -2727,11 +2744,11 @@ class IsMember : public Validator { // Get the type of the contained item - requires a container have ::value_type // if the type does not have first_type and second_type, these are both value_type - using element_t = typename detail::element_type::type; // Removes (smart) pointers if needed - using item_t = typename detail::pair_adaptor::first_type; // Is value_type if not a map + using element_t = typename detail::element_type::type; // Removes (smart) pointers if needed + using item_t = typename detail::pair_adaptor::first_type; // Is value_type if not a map - using local_item_t = typename IsMemberType::type; // This will convert bad types to good ones - // (const char * to std::string) + using local_item_t = typename IsMemberType::type; // This will convert bad types to good ones + // (const char * to std::string) // Make a local copy of the filter function, using a std::function if not one already std::function filter_fn = filter_function; @@ -2744,7 +2761,7 @@ class IsMember : public Validator { func_ = [set, filter_fn](std::string &input) { local_item_t b; if(!detail::lexical_cast(input, b)) { - throw ValidationError(input); // name is added later + throw ValidationError(input); // name is added later } if(filter_fn) { b = filter_fn(b); @@ -2800,10 +2817,10 @@ class Transformer : public Validator { "mapping must produce value pairs"); // Get the type of the contained item - requires a container have ::value_type // if the type does not have first_type and second_type, these are both value_type - using element_t = typename detail::element_type::type; // Removes (smart) pointers if needed - using item_t = typename detail::pair_adaptor::first_type; // Is value_type if not a map - using local_item_t = typename IsMemberType::type; // This will convert bad types to good ones - // (const char * to std::string) + using element_t = typename detail::element_type::type; // Removes (smart) pointers if needed + using item_t = typename detail::pair_adaptor::first_type; // Is value_type if not a map + using local_item_t = typename IsMemberType::type; // Will convert bad types to good ones + // (const char * to std::string) // Make a local copy of the filter function, using a std::function if not one already std::function filter_fn = filter_function; @@ -2858,12 +2875,11 @@ class CheckedTransformer : public Validator { "mapping must produce value pairs"); // Get the type of the contained item - requires a container have ::value_type // if the type does not have first_type and second_type, these are both value_type - using element_t = typename detail::element_type::type; // Removes (smart) pointers if needed - using item_t = typename detail::pair_adaptor::first_type; // Is value_type if not a map - using local_item_t = typename IsMemberType::type; // This will convert bad types to good ones - // (const char * to std::string) - using iteration_type_t = typename detail::pair_adaptor::value_type; // the type of the object pair // - // the type of the object pair + using element_t = typename detail::element_type::type; // Removes (smart) pointers if needed + using item_t = typename detail::pair_adaptor::first_type; // Is value_type if not a map + using local_item_t = typename IsMemberType::type; // Will convert bad types to good ones + // (const char * to std::string) + using iteration_type_t = typename detail::pair_adaptor::value_type; // the type of the object pair // Make a local copy of the filter function, using a std::function if not one already std::function filter_fn = filter_function; @@ -3071,7 +3087,7 @@ class AsNumberWithUnit : public Validator { /// "2 EiB" => 2^61 // Units up to exibyte are supported class AsSizeValue : public AsNumberWithUnit { public: - using result_t = uint64_t; + using result_t = std::uint64_t; /// If kb_is_1000 is true, /// interpret 'kb', 'k' as 1000 and 'kib', 'ki' as 1024 @@ -3147,12 +3163,12 @@ inline std::pair split_program_name(std::string comman return vals; } -} // namespace detail +} // namespace detail /// @} -} // namespace CLI +} // namespace CLI -// From CLI/FormatterFwd.hpp: +// From FormatterFwd.hpp: namespace CLI { @@ -3165,9 +3181,9 @@ class App; /// the second argument. enum class AppFormatMode { - Normal, //< The normal, detailed help - All, //< A fully expanded help - Sub, //< Used when printed as part of expanded subcommand + Normal, ///< The normal, detailed help + All, ///< A fully expanded help + Sub, ///< Used when printed as part of expanded subcommand }; /// This is the minimum requirements to run a formatter. @@ -3196,7 +3212,7 @@ class FormatterBase { FormatterBase(FormatterBase &&) = default; /// Adding a destructor in this form to work around bug in GCC 4.7 - virtual ~FormatterBase() noexcept {} // NOLINT(modernize-use-equals-default) + virtual ~FormatterBase() noexcept {} // NOLINT(modernize-use-equals-default) /// This is the key method that puts together help virtual std::string make_help(const App *, std::string, AppFormatMode) const = 0; @@ -3241,7 +3257,7 @@ class FormatterLambda final : public FormatterBase { explicit FormatterLambda(funct_t funct) : lambda_(std::move(funct)) {} /// Adding a destructor (mostly to make GCC 4.7 happy) - ~FormatterLambda() noexcept override {} // NOLINT(modernize-use-equals-default) + ~FormatterLambda() noexcept override {} // NOLINT(modernize-use-equals-default) /// This will simply call the lambda function std::string make_help(const App *app, std::string name, AppFormatMode mode) const override { @@ -3318,9 +3334,9 @@ class Formatter : public FormatterBase { ///@} }; -} // namespace CLI +} // namespace CLI -// From CLI/Option.hpp: +// From Option.hpp: namespace CLI { @@ -3334,11 +3350,11 @@ class App; using Option_p = std::unique_ptr