pants.ini

# Pants config for Foursquare Fsq.io

[GLOBAL]
print_exception_stacktrace: True
enable_pantsd: False

# We use os.environ in a few RPM BUILD files that want to consume the shared version from env.sh.
# TODO(mateo): Consider a macro for environmental variables upstream.
build_file_imports: allow
kill_nailguns: True
# cc-shapefile hosting for Fsq.io users.
binaries_baseurls: +[
    'https://github.com/foursquare/cc-shapefiles/raw/master/blobs/',
  ]

# The pants_version value is only used in Fsq.io, although checked on each run everywhere.
# TODO(mateo): Move just the version declaration to a deployed file so we can unit test the
# rest of this config against prod.
pants_version: 1.7.0

backend_packages: +[
    'pants.backend.docgen',
    'pants.contrib.confluence',
    'pants.contrib.node',
    'fsqio.pants.buildgen.core',
    'fsqio.pants.buildgen.jvm',
    'fsqio.pants.buildgen.python',
    'fsqio.pants.buildgen.spindle',
    'fsqio.pants.export',
    'fsqio.pants.ivy',
    'fsqio.pants.node',
    'fsqio.pants.publish',
    'fsqio.pants.python',
    'fsqio.pants.rpmbuild',
    'fsqio.pants.spindle',
    'fsqio.pants.tags',
    'fsqio.pants.wiki',
  ]

# Mixed into all cache keys. Bump this to invalidate all existing artifacts.
# We are using timestamps for cache keys. To bump it run `date +%Y-%m-%d:%H:%M:%S` and paste in the output.
cache_key_gen_version: '2021-03-01:23:12:08'

# The build_ignore_patterns uses .gitignore syntax.
build_ignore: [
    '.*/',
    '/.pants.d',
    '.git',
    '*.orig',
    '*.rej',
    '*.pyc',
    '.local_artifact_cache',
    '.pants.bootstrap',
    '.pants.d',
    '.pantsenv',
    '.pids',
    '.pvenvs',
    '.wheelhouse',
    'environ.sh',
    '/api',
    '/dependencies',
    '/dist',
    '/etc',
    '/log',
    '/mongo-testdb',
    '/out',
    '/scripts',
    '/target',
    'node_modules/',
    'python_cache/',
  ]


[DEFAULT]

pants_workdir: %(buildroot)s/.pants.d
pants_supportdir: %(buildroot)s/build-support
pants_distdir: %(buildroot)s/dist

_base_jvm_options: [
  '-Duser.timezone=UTC',
  '-Dfile.encoding=UTF-8',
  '-Djava.awt.headless=true',
  '-XX:+AggressiveOpts',
  '-XX:+UseCompressedOops',
  '-XX:ReservedCodeCacheSize=512m',
  '-server',
  ]

# Used as the base jvm runtime options during development (i.e. not during compile, for 'run', 'repl', 'test', etc).
_dev_jvm_options: %(_base_jvm_options)s + [
  '-Xss8m',
  '-noverify',
  ]

# Overrides *all* options named "option" - this was the only effective way to override the defaults.
jvm_options: %(_base_jvm_options)s + ['-Xmx2g', '-Xss2m']
options: %(_base_jvm_options)s + ['-Xmx2g', '-Xss2m']

###
#
# Default Values for Pants Options
# These redefine the default value for all Pants options with the same name, regardles of scope.
#
###
# NOTE: These are matched by option *name* and it is easy to accidentally override more than you expected!


#### Pants Task Options

[bootstrap.global-jar-dependency-management]
# With this True, the global resolution will include the jar_library targets from any BUILD files in the buildroot.
include_buildroot: True


[jar-dependency-management]
default_target: 3rdparty:fsqio-management


[scala-platform]
version: custom
suffix_version: 2.11


[jvm-distributions]
minimum_version: 1.8.0
maximum_version: 1.8.999


[jvm-platform]
default_platform: java8
platforms: {
    'java8': {'source': '8', 'target': '8', 'args': [] },
  }


[jvm.run.jvm]
options: %(_dev_jvm_options)s + [
  '-Xmx4g',
  ]


[jvm.repl.scala]
options: %(_dev_jvm_options)s + [
  '-Xmx4g',
  '-Dscala.usejavacp=true',
  '-DconsoleMode=true',
  ]


[jvm.test.junit]
options: %(_dev_jvm_options)s + [
  "-Xmx4g",
  "-Drun.mode=test",
  '-Dscala.usejavacp=true',
  "-Djava.security.manager",
  "-Djava.security.policy=%(buildroot)s/test/jvm/fsqio.tests.policy",
  ]


[reporting]
# This cuts down on the verbosity of pants output
# TODO(mateo): This makes the day-to-day output much cleaner but also suppresses almost all the useful -ldebug output. Needs a happy medium.
console_label_format: {
    'COMPILER' : 'SUPPRESS',
  }
console_tool_output_format: {
    'COMPILER' : 'SUPPRESS',
  }


[compile.zinc]
worker_count: 4
# Unneeded since we use buildgen.
unused_deps: ignore
jvm_options: %(_base_jvm_options)s + [
  '-XX:+UseConcMarkSweepGC',
  '-XX:ParallelGCThreads=2',
  '-Dzinc.analysis.cache.limit=5000',
  '-Xmx6g',
  '-Xss8m',
  ]

# TODO(mateo): Audit against upstream and deprecate.
warning_args: []
no_warning_args: [
    '-S-nowarn',
  ]

args: [
    '-S-encoding',
    '-SUTF-8',
    '-S-deprecation',
    '-S-unchecked',
    '-S-feature',
    '-S-Xfatal-warnings',
    '-S-g:vars',
    '-S-language:existentials',
    '-S-language:implicitConversions',
    '-S-language:reflectiveCalls',
    '-S-language:postfixOps',
    '-S-language:higherKinds',
    '-S-target:jvm-1.8',
  ]
incremental: True


[repl.scala]
main: ammonite.Main


[binary.dup]
skip: True


[test.junit]
parallel_threads: 4
use_experimental_runner: True
output_mode: FAILURE_ONLY
default_concurrency: PARALLEL_CLASSES_AND_METHODS

# Chroot and no-fast are required for caching test results.
chroot: True
fast: False


[clean-all]
async: True


[bundle.jvm]
use_basename_prefix: True


[thrift]
version: 0.9.1


[build-spindle]
spindle_codegen_binary: src/jvm/io/fsq/spindle/codegen:spindle


[gen.spindle]
jvm_options: %(_base_jvm_options)s + ['-Xmx4g']
scala_ssp_template: src/resources/io/fsq/ssp/codegen/scala
spindle_codegen_binary: src/jvm/io/fsq/spindle/codegen:spindle
runtime_dependency: [
    '3rdparty:finagle',
    '3rdparty:thrift',
    'src/jvm/io/fsq/spindle/runtime',
  ]
write_annotations_json: true


[ide-gen.spindle-stubs]
jvm_options: %(_base_jvm_options)s + ['-Xmx4g']
scala_ssp_template: src/resources/io/fsq/ssp/codegen/scalainterface
spindle_codegen_binary: src/jvm/io/fsq/spindle/codegen:spindle
runtime_dependency: [
    '3rdparty:finagle',
    '3rdparty:thrift',
    'src/jvm/io/fsq/spindle/runtime',
  ]
# NOTE(mateo): This is the Fsq.io path, should work with internal users to migrate to this and deprecate surplus path.
stub_output_path: %(buildroot)s/scripts/fsqio/IntelliJ/spindle_stubs/


[lint.scalafmt]
skip=True


[fmt.scalafmt]
configuration: %(buildroot)s/build-support/fsqio/scalafmt/config


[fmt.scalafix]
skip=True


[test.pytest]
timeouts: true
# Need this timeout for some luigi auto tests.
timeout_default: 300
# Overridden here because we define a jvm-related 'options' in our default section.
options=[]
# Chroot and no-fast are required for caching test results.
chroot: True
fast: False

[test.pytest-prep]
requirements: ['more-itertools<6.0.0', 'attrs<19.2.0']


[repl.py]
# NOTE(mateo): Please keep the ipython version in sync with 3rdparty/python/requirements.txt.
ipython_requirements: ['ipython==5.0.0']
ipython: True


[mypy]
config_file: %(buildroot)s/build-support/fsqio/mypy/mypy.ini


[buildgen]
source_dirs: [
    'src',
    '3rdparty',
  ]
test_dirs: [
    'test',
  ]

target_allowlist: [
  'java_tests',
  'jsonc_config',
  'jsonc_library',
  'junit_tests',
  'python_library',
  'python_tests',
  'scala_library',
  'scala_record_library',
  'scala_soy_library',
  'scalac_plugin',
  'spindle_thrift_library'
  ]

managed_dependency_aliases: [
  'jar_library',
  'jsonc_config',
  'jsonc_library',
  'python_library',
  'python_requirement_library',
  'scala_library',
  'scala_soy_library',
  'scala_record_library',
  ]

buildgen_target_bags: [
    'src/thrift/io/fsq:all_thrift',
  ]


[buildgen.python]
fatal: True

first_party_packages = [
    'fsqio',
    'fsqio_test,'
  ]

ignored_prefixes: [
    'argparse',    # builtin, excluded because the 3rdparty version is installed transitively.
    'geojson',     # twofishes script in src/jvm.
    'lvm',         # linuxonly
    'shapely',     # twofishes script - requires c headers and not in virtualenv
  ]

# Surprised to see that py-zipkin is required here.
third_party_map: {
    'confluent_kafka': {
      'DEFAULT': '3rdparty/python:confluent-kafka',
    },
    'enum': {
      'DEFAULT': '3rdparty/python:enum34',
    },
    'gen': {
      'apache': {
        'aurora': '3rdparty/python:apache.aurora.thrift',
        'thermos': '3rdparty/python:apache.aurora.thermos',
        },
      },
    'github3': {
      'DEFAULT': '3rdparty/python:github3.py',
    },
    'pants': {
      'DEFAULT': '3rdparty/python:pantsbuild.pants',
      'contrib': {
        'confluence': '3rdparty/python:pantsbuild.pants.contrib.confluence',
        'node': '3rdparty/python:pantsbuild.pants.contrib.node',
      },
    },
    'pants_test': {
      'DEFAULT': '3rdparty/python:pantsbuild.pants.testinfra',
    },
    'py_zipkin': {
      'DEFAULT': '3rdparty/python:py-zipkin',
    },
    'gnupg': {
      'DEFAULT': '3rdparty/python:python-gnupg',
    },
    'twitter': {
      'common': {
        'thrift': '3rdparty/python:twitter.common.service-thrift',
        },
      },
  }

force_third_party: +['gnupg']


[map-scala-library-used-addresses]
third_party_map_file: src/resources/io/fsq/buildgen/third_party_map.ini


[buildgen.publish]
# Toggle on to enable provides support.
skip: True

opt_out_paths: [
    # Opting out because testlibs should be proven useful to export.
    # Requires adding `test/` to repo and metadata map to support.
    'test/'
  ]

exportable_target_aliases: [
    'java_library',
    'scala_library',
    'spindle_thrift_library',
    'scala_record_library',
  ]

# NOTE(mateo): These use ExportableJvmTarget as a base and possibly could be published.
# But doing so seems pathological. This disables even attempting it until the need is recognized.
unexportable_target_aliases: [
    'scala_js_library',
    'scala_soy_library',
    'junit_tests',
  ]

# Repository string must be defined in the publish register.py. See publish documentation for info.
repo_map: {
    'src/jvm/io/fsq': 'oss_sonatype_repo',
    'src/thrift/io/fsq': 'oss_sonatype_repo',
  }

publication_metadata_map: {
    'src/jvm/io/fsq': 'io_fsq_library("Fsq.io common libraries, from Foursquare.")',
    'src/thrift/io/fsq': 'io_fsq_library("Fsq.io common libraries, from Foursquare.")'
  }


[tag]
# Currently overridden internally, possible bug in dict_option.
by_prefix:  +{
    '3rdparty': ['exempt'],
    'test': ['tests'],
    'verification': ['tests', 'integration_tests'],
    'src': ['dependencies_cannot_have:tests'],

    'build-support/fsqio':  ['opensource', 'dependencies_must_have:opensource'],
    'src/jvm/com/twitter/ostrich': ['opensource', 'dependencies_must_have:opensource'],
    'src/jvm/io/fsq': ['opensource', 'dependencies_must_have:opensource'],
    'src/resources/io/fsq': ['opensource', 'dependencies_must_have:opensource'],
    'src/python/fsqio':  ['opensource', 'dependencies_must_have:opensource'],
    'src/thrift/io/fsq': ['opensource', 'dependencies_must_have:opensource'],
    'test/jvm/io/fsq': ['opensource', 'dependencies_must_have:opensource'],
    'test/python/fsqio':  ['opensource', 'dependencies_must_have:opensource'],
    'test/thrift/io/fsq': ['opensource', 'dependencies_must_have:opensource'],

    # Tests can rely on the spindle_codegen, but src dependencies are banned.
    'test': ['spindle_codegen'],

    # NOTE(mateo): Another isolation case, where this is a proxy for 'dependencies_must_have_one_of: rogue, fscommon'.
    # TODO(jacob): We have this now, fix
    'src/jvm/io/fsq/common': ['fscommon', 'rogue'],
    'src/jvm/io/fsq/field': ['fscommon', 'rogue'],
  }


[resolve.ivy]
disable_reports: True
resolution_report_outdir: 3rdparty/reports/jvm
global_excludes: [
    {"org": "org.slf4j", "name": "slf4j-jdk14"},
    {"org": "org.slf4j", "name": "slf4j-log4j12"},
    {"org": "org.slf4j", "name": "slf4j-simple"},
    {"org": "log4j", "name": "log4j"},
    {"org": "log4j", "name": "apache-log4j-extras"},
    {"org": "org.apache.logging.log4j", "name": "log4j-core"},
    {"org": "org.apache.logging.log4j", "name": "log4j-api"},
    {"org": "org.mongodb", "name": "mongo-java-driver"},
    {"org": "com.esotericsoftware.kryo", "name": "kryo"},
    {"org": "com.esotericsoftware", "name": "reflectasm"},
    {"org": "org.apache.hadoop", "name": "hadoop-client"},
    {"org": "org.apache.hadoop", "name": "hadoop-core"},
    # NOTE(jacob): Depending on the monolithic netty jar rather than its individual
    #   modules causes all kinds of headaches with other libraries that try to do the
    #   right thing and only pull in what they need. There is some talk in the netty
    #   project of making the monolith an empty wrapper that pulls in all the module jars
    #   (see https://github.com/netty/netty/issues/4671 for details), but for now it
    #   bundles duplicate classes and we have to sort out the conflict. Our dep report
    #   lists spark as the only problem child pulling this in, but just one is enough.
    #
    #   What we do here is exclude the monolith, and instead declare a jar_library
    #   containing the individual netty modules we need. If you add a new 3rdparty jar
    #   with a dependency on netty-all, you may see classloading issues if it uses code
    #   from a module we aren't pulling in. It is up to you to identify the missing
    #   module, add it to our netty4 jar_library, and then add a dependency from your
    #   new jar_library to our netty4 jar_library.
    #
    #   Note also included on the netty jar_library in 3rdparty/BUILD.opensource.
    {"org": "io.netty", "name": "netty-all"},
    {"org": "org.jruby", "name": "jruby-complete"},
    # Twitter publishes modules with these direct dependencies - but they are internal only.
    {"org": "com.twitter", "name": "finagle-core-java"},
    {"org": "com.twitter", "name": "libthrift"},
    {"org": "com.twitter", "name": "util-core-java"},
    {"org": "com.twitter", "name": "zookeeper"},
    # NOTE: This org.apache.zookeeper#zookeeper-client is also an internal twitter thing, somehow.
    {"org": "org.apache.zookeeper", "name": "zookeeper-client"},
    # Hadoop LZO compression is also provided by twitter.
    {"org": "com.hadoop.gplcompression", "name": "hadoop-lzo"},
  ]


[cache.bundle]
# These are huge artifacts, but if bundle is upstream of your target, a few artifacts makes a difference.
max_entries_per_target = 1


[cache.binary]
# These are huge artifacts, but if bundle is upstream of your target, a few artifacts makes a difference.
max_entries_per_target = 1


[cache.webpack.webpack-resolve]
# NOTE(mateo): This option is rolled into the task fingerprint - if it doesn't match the CI's config, the resulting
# artifacts will have mismatched cache_keys.
# Symlinks are not followed because node allows hanging symlinks as part of its fault-tolerant resolution process.
dereference_symlinks: False