diff options
| -rw-r--r-- | dev-db/scylla/Manifest | 18 | ||||
| -rw-r--r-- | dev-db/scylla/files/3.1.2-fix-read-amplification.patch | 13 | ||||
| -rw-r--r-- | dev-db/scylla/scylla-3.1.2-r1.ebuild | 254 |
3 files changed, 277 insertions, 8 deletions
diff --git a/dev-db/scylla/Manifest b/dev-db/scylla/Manifest index 6a7d9ee..cf489ed 100644 --- a/dev-db/scylla/Manifest +++ b/dev-db/scylla/Manifest @@ -6,6 +6,7 @@ AUX 0001-thrift-support-version-0.11-after-THRIFT-2221.patch 40277 BLAKE2B 4296b AUX 10-scylla.conf 64 BLAKE2B 31ce1b2173214bcac351b71cf19e5506ab08cefbca446ce23ad753c98c5e4fd7fbb89b0f79703ea4dcf1f6233e7929268365eeaab1b4e2e7b31553aa7928a969 SHA512 c24b3ec2f0ea3b1ad0792c05a8eb044b3a157862b59759e020586ae872f9789a558d423fe1dff1a8c6d8342049cf4d919faf31a1439a6a1c69af915365b4ec82 AUX 3.1-revert-64-bit-gc_clock.patch 1132 BLAKE2B 7ed59d5f898babc4940bcd10a8aaf467545c80fd8c7417e481d80abc07fc27b4859396cafa5b1d1da4d386f8c490e6f417c0ce63064eea7177ce869869387480 SHA512 8ff0019080687993939fb5c0bb24945214799dfe0942982d8154769155159c02bc75e448978343c1b08a162a6670bede3721247a70c6fabfe1cd9730dbf4f77a AUX 3.1-thrift-support.patch 39980 BLAKE2B 7de8bde052186b8a62ea83b7082e9828d4c320ed86381a93d5136906ba014631dc3b5e2a2b1dfd33bec21c6bd7d685fa3f1f270c97f233934cca24473d0e52b5 SHA512 d33e50dbef01dfd4845a093c160b5cbedc658770c1b5a5de063c56b43df0731329fe0cdb3ddfa5d72020ab25573682e4d14a3c39ad366a208694062a3fd49155 +AUX 3.1.2-fix-read-amplification.patch 962 BLAKE2B bacd95cf99097254a3aca2a034ac38c6133039ab0487cb2880ef4fb3ed960c066bf96470a4c07cf800cd01810b7841c9740ea8204aa0ecd73b2b71489dfcc591 SHA512 59b8646378cc6ec9468920093169f1f537e9855e6b1f4a83f4abaaf15c1c2c66566877d3bae88b4f5b75ab0fb8958527f9a13f13f6a98e839c56bb0418b20c27 AUX boost-1.65.0-icl-ttp-matching.patch 1427 BLAKE2B de9c77253963656f85ed2d05babd600bc5b59b567bc889ed815dfba97f2a1334df48dbb1e67879dea1419fb439c513c54a66531d4e8ec28a7848385f70471334 SHA512 aaa7959df631c17a80486e5b6724d76bf2193694f5559443350c74567506f36c133f740adcbb1b2d3a5a2760730f498f0fd655ec6eff6d20393b6e49549c6ec8 AUX fix-fmt-3.5.0-compilation.patch 3785 BLAKE2B 3cd23ccffe1ec33d5f7e2c6c3583631b294fc0e6d36e49eef582598fd168968805dd7ca4cc786c02cb20ae1e758a814e1b761dbd41a86330906cd4b5af5d6666 SHA512 d7606559fb2aef9ea009e8c417397dcc94d0c708da8283d4c1c6393c059ecca50d9806cfb19d1fe69b8d7898da2b8d194fbc8b8dd1aa9c12e0d6120d52bc2518 AUX scylla-server.confd 396 BLAKE2B dc522b308985e51c24449a333d793eaa777a74b8b5594f1ff4a092aae003202791019a255ea478c4e794a83ca0e01d2c49aac3d267501624f461fa3edfe1c41f SHA512 51739cdeedaedb1a73a345ec6a4a2761d29fac1b9cf61f02937559aed43f02112109464254213118d40af1fb7e8491c22f8cb88908c9c88523dfe1b78b8430c7 @@ -32,20 +33,21 @@ EBUILD scylla-3.1.0.ebuild 7051 BLAKE2B 282b4221655c0156d3a39f7245c8b6023a1f4352 EBUILD scylla-3.1.1-r1.ebuild 7571 BLAKE2B 999faabaf0a6d2b8de9f555c0170288a1add0ac69d05afd3ab2b1976deb6e560e3046edede9caefcf56f38cd9995aa0e40f184ba14e15cec35b1f3ccf47e3b7c SHA512 fb6442b55e8152ed6c99f7473dd3b5075521f46e23d3124bfa01a13d6d134df872273c9eae207d96d09a7056f2496867c74deca37090c762c1af026708e55394 EBUILD scylla-3.1.1-r2.ebuild 7630 BLAKE2B 3593fb08d50353daae99dcabfca109a4abd06853bcdff91b7613faf37fab0bc538413957a0aca3015be86748ab9dc0b10defd62115704399b189715695fce4ba SHA512 b5c6b9263b292a2830ea209676ee2dfddaf44f7b22be61bbdaae7c1a776246bdea92e4780dc14e1df68428cd95a8a777eea41efc27c9203e44d4a7f73b354e5e EBUILD scylla-3.1.1.ebuild 7505 BLAKE2B 1f3e068c6688c9f3a420102da25966a142dfd3be1f90d94c77c2d744274df4d0eaa82e4af1eeef53d7b6594d940f188263428befd018a4da9aa9ef0896f5799b SHA512 8769a7fd5f69038f958eddd38a70da8cdc76d606638422224394164be09b4b3c0f72c5aac807bcc199a3f56d05b1c3baf48d7dfd1523602b0aca00681fa8a0c4 +EBUILD scylla-3.1.2-r1.ebuild 7680 BLAKE2B e19a665cce5469d956f45e60aa5dbcedd9e7b25ff8d81a3877306b338befa5741f68cd9c805b39fded300560d5999a0d782e36a91f3adb6974388b88baf53f61 SHA512 f3c0ea58ace690d68bbc0646828183a93730ebbede14b7200391aa967f0249b9f8a55c8146b407b5834b4401bb12b192ddbb61c9115148997fc56792d5cedefc EBUILD scylla-3.1.2.ebuild 7630 BLAKE2B 3593fb08d50353daae99dcabfca109a4abd06853bcdff91b7613faf37fab0bc538413957a0aca3015be86748ab9dc0b10defd62115704399b189715695fce4ba SHA512 b5c6b9263b292a2830ea209676ee2dfddaf44f7b22be61bbdaae7c1a776246bdea92e4780dc14e1df68428cd95a8a777eea41efc27c9203e44d4a7f73b354e5e EBUILD scylla-9999.ebuild 6833 BLAKE2B c21b0db85ca5b2dbe90a7523b739fa6614d40fd0264d8cd5bf77f1c1b37b31f1f042c946b76d5c1e74907badf931c188f7f5112105089658ff6acfc80f440021 SHA512 e5237eddd44fa0f4f88a6aa4bd2d6015dc6c61597636e03957ea932cce240de05533e223aae0abb68d37d16c7734e64267f2b77aedb3913c24b6b73210629972 MISC metadata.xml 249 BLAKE2B 40afa233e18ae5678e52194be8b1235d8deda4810e55b33ea192ded6b09a158f1bc941544f90a1b9e396db00bc0fe277c072d267568473ae4aa2dec8bc5c2f89 SHA512 6eae78a65db44c8f7ef2d20c10c8612cd2072c0204c6294798e4bf94f1283c684c22d1ad28e0f68e78e82a0c60526d6786492c9dacc438d83405218e51c1bf9c -----BEGIN PGP SIGNATURE----- Version: GnuPG v2 -iQGTBAEBCAB9FiEEB7NG2KqNbQw8gqQ7Uf8j8ayXdy8FAl3uFjlfFIAAAAAALgAo +iQGTBAEBCAB9FiEEB7NG2KqNbQw8gqQ7Uf8j8ayXdy8FAl3vhTRfFIAAAAAALgAo aXNzdWVyLWZwckBub3RhdGlvbnMub3BlbnBncC5maWZ0aGhvcnNlbWFuLm5ldDA3 QjM0NkQ4QUE4RDZEMEMzQzgyQTQzQjUxRkYyM0YxQUM5Nzc3MkYACgkQUf8j8ayX -dy9/9QgAv1pGnXHqxHx69Z/l2KFTxJkZiIS6fo+oTVfD/ME3kzKTBFwVvP+Wn0UU -OsOT7n04Q0Pmyq7yR+xPQIfE5SRjIM1PmVFs4Wu7bKW84VhdB8F5gUaLz/vRVmJh -3M9zwF99PidjkSJJfEbXR1CXMQ/s5aCE2s5oUV4MG7Uqkuh7onMgEyuvw1G906u4 -e8W5Eld8CngqYhFjvu9+NRO++7SCwRwzm4O6TrQwDMgiX19FgTItX+iArYQkUuEb -roUtwu/czuVA6hvfLoF/N56FD96+v1Sxiy3OZsMj5NWw24Zp8W5RFaCdu5mqBr25 -HXnYN+n5CZsWlHRjtjIH+EJ4wmd1fQ== -=P05N +dy/N5Af+K3fbAwbVedEEMoAqr8Hyd02aWJ4IWADrl0tSFUH2pIIiCHIZQwXA8E8U +0iGkO7dWF8Ej4Cj/d+mbqFqMZ8+wr6y84RXR1dZ9Z+7acyzy/fDb2O0fyOVddFt4 +ELPY6KVYR9PPbh+WbkwYBlFJ+wmhH483SF4Y6mnsgnQnoMPwXR3ypuOxWC3yAbvF +CAtfKL0yugpKFbIiZ9GXKy3+XsHNKRqazsiaoxN6TzN7NJGXie+e6uS+rvtPg6SA +G4BHNuWF5iE5Hzc2QfEKfRZLG0k7WYc3uLkWzf4qLJkAknJWxmR50o2sy5bVf03W +p+SVoSWG69jVQSngmse+pWiIBw5M/A== +=p1ER -----END PGP SIGNATURE----- diff --git a/dev-db/scylla/files/3.1.2-fix-read-amplification.patch b/dev-db/scylla/files/3.1.2-fix-read-amplification.patch new file mode 100644 index 0000000..c0dceef --- /dev/null +++ b/dev-db/scylla/files/3.1.2-fix-read-amplification.patch @@ -0,0 +1,13 @@ +diff --git a/table.cc b/table.cc +index eec65689f2..596c312854 100644 +--- a/table.cc ++++ b/table.cc +@@ -2518,7 +2518,7 @@ future<row_locker::lock_holder> table::do_push_view_replica_updates(const schema + std::move(slice), + std::move(m), + [base, views = std::move(views), lock = std::move(lock), this, timeout, source = std::move(source), &io_priority] (auto& pk, auto& slice, auto& m) mutable { +- auto reader = source.make_reader(base, pk, slice, io_priority); ++ auto reader = source.make_reader(base, pk, slice, io_priority, nullptr, streamed_mutation::forwarding::no, mutation_reader::forwarding::no); + return this->generate_and_propagate_view_updates(base, std::move(views), std::move(m), std::move(reader)).then([lock = std::move(lock)] () mutable { + // return the local partition/row lock we have taken so it + // remains locked until the caller is done modifying this diff --git a/dev-db/scylla/scylla-3.1.2-r1.ebuild b/dev-db/scylla/scylla-3.1.2-r1.ebuild new file mode 100644 index 0000000..86e46aa --- /dev/null +++ b/dev-db/scylla/scylla-3.1.2-r1.ebuild @@ -0,0 +1,254 @@ +# Copyright 1999-2019 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +EAPI=6 + +MY_PV="${PV/_beta/}" +EGIT_CLONE_TYPE="mirror" +EGIT_COMMIT="scylla-${PV}" +EGIT_REPO_URI="https://github.com/scylladb/scylla.git" +inherit git-r3 + +PYTHON_COMPAT=( python3_{4,5,6} ) + +inherit autotools flag-o-matic linux-info python-r1 toolchain-funcs systemd user + +DESCRIPTION="NoSQL data store using the seastar framework, compatible with Apache Cassandra" +HOMEPAGE="http://scylladb.com/" + +KEYWORDS="~amd64" +LICENSE="AGPL-3" +SLOT="0" +IUSE="doc systemd" + +# NOTE: +# if you want to debug using backtraces, enable the 'splitdebug' FEATURE: +# https://wiki.gentoo.org/wiki/Project:Quality_Assurance/Backtraces +# +# then check out: +# https://github.com/scylladb/scylla/wiki/How-to-resolve-backtrace + +RESTRICT="test" + +RDEPEND=" + >=dev-libs/thrift-0.11.0 + <dev-util/ragel-7.0 + <sys-apps/hwloc-2.0.0 + ~app-admin/scylla-jmx-${PV} + ~app-admin/scylla-tools-${PV} + >=virtual/jdk-1.8.0:* + app-arch/lz4 + app-arch/snappy + app-arch/zstd + dev-cpp/antlr-cpp:3.5 + dev-cpp/yaml-cpp + dev-java/antlr:3.5 + ~dev-libs/boost-1.65.0[icu] + dev-libs/crypto++ + dev-libs/jsoncpp + dev-libs/libaio + >=dev-libs/libfmt-3.2.1 + dev-libs/libxml2 + dev-libs/protobuf + dev-python/psutil[${PYTHON_USEDEP}] + dev-python/pyparsing[${PYTHON_USEDEP}] + dev-python/pystache[${PYTHON_USEDEP}] + dev-python/pyudev[${PYTHON_USEDEP}] + >=dev-python/pyyaml-5.1[${PYTHON_USEDEP}] + dev-python/requests[${PYTHON_USEDEP}] + dev-python/urwid[${PYTHON_USEDEP}] + dev-util/systemtap + net-libs/gnutls + net-misc/lksctp-tools + sys-apps/ethtool + sys-fs/xfsprogs + sys-libs/libunwind + sys-libs/zlib + sys-process/numactl + x11-libs/libpciaccess + systemd? ( sys-apps/systemd ) +" +DEPEND="${RDEPEND} + >=sys-kernel/linux-headers-3.5 + dev-util/ninja +" + +# Discussion about kernel configuration: +# https://groups.google.com/forum/#!topic/scylladb-dev/qJu2zrryv-s +CONFIG_CHECK="~NUMA_BALANCING ~SYN_COOKIES ~TRANSPARENT_HUGEPAGE" +ERROR_NUMA_BALANCING="${PN} recommends support for Memory placement aware NUMA scheduler (NUMA_BALANCING)." +ERROR_SYN_COOKIES="${PN} recommends support for TCP syncookie (SYN_COOKIES)." +ERROR_TRANSPARENT_HUGEPAGE="${PN} recommends support for Transparent Hugepage (TRANSPARENT_HUGEPAGE)." + +# NOTE: maybe later depending on upstream energy, support DPDK +# For DPDK, removed HUGETLBFS PROC_PAGE_MONITOR UIO_PCI_GENERIC in favor of VFIO +# CONFIG_CHECK="~NUMA_BALANCING ~SYN_COOKIES ~TRANSPARENT_HUGEPAGE ~VFIO" +# ERROR_VFIO="${PN} running with DPDK recommends support for Non-Privileged userspace driver framework (VFIO)." + +DOCS=( LICENSE.AGPL NOTICE.txt ORIGIN README.md README-DPDK.md ) +PATCHES=( + "${FILESDIR}/fix-fmt-3.5.0-compilation.patch" + "${FILESDIR}/3.1-thrift-support.patch" + "${FILESDIR}/v1-0001-repair-Add-a-timer-for-repair-reader.patch" + "${FILESDIR}/0001-repair-Repair-tables-in-parallel.patch" + "${FILESDIR}/3.1.2-fix-read-amplification.patch" +) + +pkg_setup() { + linux-info_pkg_setup + enewgroup scylla + enewuser scylla -1 -1 /var/lib/${PN} scylla +} + +src_prepare() { + default + + # set version + local git_commit=$(git log --pretty=format:'%h' -n 1) + echo "${PV}-${git_commit}" > version + + # fix jsoncpp detection + sed -e 's@json/json.h@jsoncpp/json/json.h@g' -i json.hh || die + + # QA: no -Werror (tho it does not propagate) + sed -e 's/\-Werror//g' -i configure.py || die + + # QA NOTE: respect -O3 as it is upstream recommended + # and because we would kill the RAM of the machine with lower optimization + # since some files can take up to 8GB of RAM to compile! + # sed -e 's/\-O3//g' -i configure.py || die + + # I don't agree with the old 4GB of RAM per job, it's more about 8GB now + sed -e 's/4000000000/8000000000/g' -i scripts/jobs || die + + # TODO: contribute me + # fix pyyaml vulnerability #659348 + # seastar/scripts/perftune.py + # dist/docker/redhat/scyllasetup.py + # dist/common/scripts/scylla_io_setup + # dist/common/scripts/scylla_fstrim + # dist/common/scripts/scylla_blocktune.py + # dist/common/scripts/scylla_util.py + # dist/common/scripts/scylla_config_get.py + find "${S}/dist" -type f -exec sed -e 's/yaml.load(/yaml.full_load(/g' -i {} \+ || die + sed -e 's/yaml.load(/yaml.safe_load(/g' -i seastar/scripts/perftune.py || die +} + +src_configure() { + python_setup + + # copied from dist/redhat/scylla.spec.mustache + # we want a package compiled with old kernel headers to + # support nowait aio if the user upgrades their kernel + if ! grep -qwr RWF_NOWAIT /usr/include/linux; then + append-cflags "-DRWF_NOWAIT=8" + fi + if ! grep -qwr aio_rw_flags /usr/include/linux; then + append-cflags "-Daio_rw_flags=aio_reserved1" + fi + + # native CPU CFLAGS are strongly enforced by upstreams, respect that + replace-cpu-flags "*" "native" + + append-cflags "-Wno-attributes -Wno-array-bounds" + + ${EPYTHON} configure.py --enable-gcc6-concepts --mode=release --with=scylla --disable-xen --c-compiler "$(tc-getCC)" --compiler "$(tc-getCXX)" --ldflags "${LDFLAGS}" --cflags "${CFLAGS}" --python ${EPYTHON} --with-antlr3 /usr/bin/antlr3.5 || die +} + +src_compile() { + # we use the provided 'scripts/jobs' to figure out how many parallel + # compilation jobs we can sustain + einfo "Compiling using $(scripts/jobs) jobs" + ninja -v build/release/scylla build/release/iotune -j$(scripts/jobs) || die +} + +src_install() { + # NOTE: + # I base myself on upstream's install.sh file + + default + + local MUSTACHE_DIST="\"debian\": true" + pystache dist/common/systemd/scylla-server.service.mustache "{ $MUSTACHE_DIST }" > build/scylla-server.service + pystache dist/common/systemd/scylla-housekeeping-daily.service.mustache "{ $MUSTACHE_DIST }" > build/scylla-housekeeping-daily.service + pystache dist/common/systemd/scylla-housekeeping-restart.service.mustache "{ $MUSTACHE_DIST }" > build/scylla-housekeeping-restart.service + + insinto /etc/default + doins dist/common/sysconfig/scylla-server + + insinto /etc/security/limits.d + doins dist/common/limits.d/scylla.conf + + insinto /etc/scylla.d + doins dist/common/scylla.d/*.conf + + insinto /etc/sysctl.d + doins dist/common/sysctl.d/*.conf + + insinto /etc/scylla + for config_file in scylla.yaml cassandra-rackdc.properties; do + doins conf/${config_file} + done + + systemd_dounit build/*.service + systemd_dounit dist/common/systemd/*.service + systemd_dounit dist/common/systemd/*.timer + + exeinto /usr/lib/scylla + doexe dist/common/scripts/* + doexe seastar/scripts/* + doexe scylla-gdb.py + + dobin build/release/iotune + dobin build/release/scylla + + insinto /usr/lib/scylla + doins dist/common/scripts/scylla_blocktune.py + + exeinto /usr/lib/scylla + doexe dist/common/scripts/scylla-blocktune + doexe dist/common/scripts/scylla-housekeeping + + insinto /etc/scylla.d + doins conf/housekeeping.cfg + + newinitd "${FILESDIR}/scylla-server.initd" ${PN}-server + newconfd "${FILESDIR}/scylla-server.confd" ${PN}-server + + dodoc -r licenses + + for x in /var/lib/${PN} /var/lib/${PN}/{data,commitlog,hints,coredump} /var/lib/scylla-housekeeping /var/log/scylla; do + keepdir "${x}" + fowners scylla:scylla "${x}" + done + + insinto /usr/lib/scylla/swagger-ui + doins -r swagger-ui/dist + + insinto /usr/lib/scylla/api + doins -r api/api-doc + + insinto /usr/lib/scylla/scyllatop + doins -r tools/scyllatop/* + fperms +x /usr/lib/scylla/scyllatop/scyllatop.py + dosym /usr/lib/scylla/scyllatop/scyllatop.py /usr/sbin/scyllatop + + for util in $(ls dist/common/sbin/); do + dosym /usr/lib/scylla/${util} /usr/sbin/${util} + done + + insinto /etc/sudoers.d + newins "${FILESDIR}"/scylla.sudoers scylla + + insinto /etc/rsyslog.d + doins "${FILESDIR}/10-scylla.conf" +} + +pkg_postinst() { + elog "You should run 'emerge --config dev-db/scylla' to finalize your ScyllaDB installation." +} + +pkg_config() { + elog "Running 'scylla_setup'..." + scylla_setup +} |