summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorUltrabug <ultrabug@gentoo.org>2019-11-28 16:59:59 +0100
committerUltrabug <ultrabug@gentoo.org>2019-11-28 16:59:59 +0100
commit4f1f8f8362b7fab0df78d76791c453cf32b53ca8 (patch)
tree1dde9fd071b67b721bfa125dbee3d097cf3f481a /dev-db/scylla
parentdev-db/scylla: add patch to fix stuck repairs (diff)
downloadultrabug-4f1f8f8362b7fab0df78d76791c453cf32b53ca8.tar.gz
ultrabug-4f1f8f8362b7fab0df78d76791c453cf32b53ca8.tar.bz2
ultrabug-4f1f8f8362b7fab0df78d76791c453cf32b53ca8.zip
dev-db/scylla: add asias patch to debug 3.1.1 on r2
Package-Manager: Portage-2.3.79, Repoman-2.3.18 Manifest-Sign-Key: 34FCB257983BA23A8CEC5EB7EB9E6405C24BFAE5 Signed-off-by: Alexys Jacob <ultrabug@gentoo.org>
Diffstat (limited to 'dev-db/scylla')
-rw-r--r--dev-db/scylla/Manifest18
-rw-r--r--dev-db/scylla/files/0001-repair-Repair-tables-in-parallel.patch44
-rw-r--r--dev-db/scylla/scylla-3.1.1-r2.ebuild253
3 files changed, 307 insertions, 8 deletions
diff --git a/dev-db/scylla/Manifest b/dev-db/scylla/Manifest
index 09bf54f..198a8f6 100644
--- a/dev-db/scylla/Manifest
+++ b/dev-db/scylla/Manifest
@@ -1,6 +1,7 @@
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA256
+AUX 0001-repair-Repair-tables-in-parallel.patch 1770 BLAKE2B 9ea285af8092a6b7f371633a6eb8049b817a4b21baf936bfb7d77bfcdbe4a00be8f98c74ede0757d8f230cde64ab8f05769d5483aefecf3bed403182368f6114 SHA512 2b73b27b12acee01dff4bf12fd7e541694acb1c5542e573f757fd0646ed7f36eb754e177fdbee0737ffefa60e44d62b970897da76052a609662c0e0f99a7d101
AUX 0001-thrift-support-version-0.11-after-THRIFT-2221.patch 40277 BLAKE2B 4296b042320db3f3669f487961e58f75403d79fbdce06a4f04b3c9e83b243b637a513cf868c618fac26df82d20bfb7de44b51c959e09d299f6a1f72e54387761 SHA512 5403833a82972be26e5d50f0557cb7a4a2985dec487be3aeb907720b5a8f638d1e3c6bfd3d06bb0c097c707135f29b137f63e77d76ba918f009d179a27dc0c91
AUX 10-scylla.conf 64 BLAKE2B 31ce1b2173214bcac351b71cf19e5506ab08cefbca446ce23ad753c98c5e4fd7fbb89b0f79703ea4dcf1f6233e7929268365eeaab1b4e2e7b31553aa7928a969 SHA512 c24b3ec2f0ea3b1ad0792c05a8eb044b3a157862b59759e020586ae872f9789a558d423fe1dff1a8c6d8342049cf4d919faf31a1439a6a1c69af915365b4ec82
AUX 3.1-revert-64-bit-gc_clock.patch 1132 BLAKE2B 7ed59d5f898babc4940bcd10a8aaf467545c80fd8c7417e481d80abc07fc27b4859396cafa5b1d1da4d386f8c490e6f417c0ce63064eea7177ce869869387480 SHA512 8ff0019080687993939fb5c0bb24945214799dfe0942982d8154769155159c02bc75e448978343c1b08a162a6670bede3721247a70c6fabfe1cd9730dbf4f77a
@@ -29,20 +30,21 @@ EBUILD scylla-3.0.8.ebuild 9111 BLAKE2B 565f5d1b42257afbfdf12ec729b7950b77b0f8d7
EBUILD scylla-3.0.9.ebuild 9218 BLAKE2B 0b2966a89c72872bebfd3dbe3bc6fea72257a40e269e812037f574491e5fa28f87b1838978251893e83cf207f6d19db63c4800783071a80d137f24cef6f72cde SHA512 1dbe1b9b4cc719c0241fd9f33f96dffbb2b3430963fc4b57e300059547d274dd3e6592db31bacc60cca783fbe862586c93efd58c8b3b6bbbb303864066d5be12
EBUILD scylla-3.1.0.ebuild 7051 BLAKE2B 282b4221655c0156d3a39f7245c8b6023a1f4352f864e51c720fbd7059435814a184743d5785a24c5b956f8201815e2ab3f211d261270f9754dc1cfc1605050b SHA512 b386d7394456070de65876c49f9976eaa487dfdf4f949555569d678396e6f18a655081c447681bce226bb8f59a5f1ee60e5e2390464e3d0bd21fd38125f8e244
EBUILD scylla-3.1.1-r1.ebuild 7571 BLAKE2B 999faabaf0a6d2b8de9f555c0170288a1add0ac69d05afd3ab2b1976deb6e560e3046edede9caefcf56f38cd9995aa0e40f184ba14e15cec35b1f3ccf47e3b7c SHA512 fb6442b55e8152ed6c99f7473dd3b5075521f46e23d3124bfa01a13d6d134df872273c9eae207d96d09a7056f2496867c74deca37090c762c1af026708e55394
+EBUILD scylla-3.1.1-r2.ebuild 7630 BLAKE2B 3593fb08d50353daae99dcabfca109a4abd06853bcdff91b7613faf37fab0bc538413957a0aca3015be86748ab9dc0b10defd62115704399b189715695fce4ba SHA512 b5c6b9263b292a2830ea209676ee2dfddaf44f7b22be61bbdaae7c1a776246bdea92e4780dc14e1df68428cd95a8a777eea41efc27c9203e44d4a7f73b354e5e
EBUILD scylla-3.1.1.ebuild 7505 BLAKE2B 1f3e068c6688c9f3a420102da25966a142dfd3be1f90d94c77c2d744274df4d0eaa82e4af1eeef53d7b6594d940f188263428befd018a4da9aa9ef0896f5799b SHA512 8769a7fd5f69038f958eddd38a70da8cdc76d606638422224394164be09b4b3c0f72c5aac807bcc199a3f56d05b1c3baf48d7dfd1523602b0aca00681fa8a0c4
EBUILD scylla-9999.ebuild 6833 BLAKE2B c21b0db85ca5b2dbe90a7523b739fa6614d40fd0264d8cd5bf77f1c1b37b31f1f042c946b76d5c1e74907badf931c188f7f5112105089658ff6acfc80f440021 SHA512 e5237eddd44fa0f4f88a6aa4bd2d6015dc6c61597636e03957ea932cce240de05533e223aae0abb68d37d16c7734e64267f2b77aedb3913c24b6b73210629972
MISC metadata.xml 249 BLAKE2B 40afa233e18ae5678e52194be8b1235d8deda4810e55b33ea192ded6b09a158f1bc941544f90a1b9e396db00bc0fe277c072d267568473ae4aa2dec8bc5c2f89 SHA512 6eae78a65db44c8f7ef2d20c10c8612cd2072c0204c6294798e4bf94f1283c684c22d1ad28e0f68e78e82a0c60526d6786492c9dacc438d83405218e51c1bf9c
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v2
-iQGTBAEBCAB9FiEEB7NG2KqNbQw8gqQ7Uf8j8ayXdy8FAl3eP05fFIAAAAAALgAo
+iQGTBAEBCAB9FiEEB7NG2KqNbQw8gqQ7Uf8j8ayXdy8FAl3f7uNfFIAAAAAALgAo
aXNzdWVyLWZwckBub3RhdGlvbnMub3BlbnBncC5maWZ0aGhvcnNlbWFuLm5ldDA3
QjM0NkQ4QUE4RDZEMEMzQzgyQTQzQjUxRkYyM0YxQUM5Nzc3MkYACgkQUf8j8ayX
-dy8fYQf+M4EfALAYpM5+f5gKxLKBz75DQllj/7cslhLYyUnnCUUO+p4N/B25vYmo
-+EuQI/8DHePRcTb8VtD83uz9XWanzTc76oJbgIKiVdy6GVyGKdbwAd/1GRqFwWXc
-5jjAkDulqLpp9xAxkwrYNJOCNBd8HiFr9VUHrNSBDPScxdk0xnK4YhPAp5NvpeY2
-kg4Foy0LgFPOtmu0QEZCpimZzgrWJpM4iHbC67DfjkcQ8fZIejFOxzUNvxpIWn9m
-mM4jg343m0wsUulFzhHYzZeRypOHtOCV8b4FYRvuwlHeKgJVL9b2/DI3ILQclefG
-cPFa7BncL1kB/4O9XSw2OSwR8h2Whw==
-=t247
+dy+gzAf/cfB9kSTb9madBq91gG7VdTiuh1S/9Tna277V2jqBCDSfqOn7FvUQLHNU
+K7k4xuu4ex1LaB0H3yNtWxUK/zSLKctQ0WXQxwd8JUVB0ya0Gfg+gy35gW2VQUbk
+zbJU9krgxhCMAvGnMS+USyBx6pDMUgDqP0loFA8e91RKW6fmlS4zxjOAMALHGvzB
+2gANmac+j1kz2bk3ctZ0/NUG6PTHcfjs9Ml8q7uVq09Nn+dJ1/eZUB6nILsFffA7
+2hj4ZWDlqFKM9GWvkwQJqLSqqF5c9o+JhaM/apEbdKTzcK7Y0Z3qmTVMW8zf5aFj
+pem2tYp3rzKc9MxwBj0R8jBLrRJYnQ==
+=LbUA
-----END PGP SIGNATURE-----
diff --git a/dev-db/scylla/files/0001-repair-Repair-tables-in-parallel.patch b/dev-db/scylla/files/0001-repair-Repair-tables-in-parallel.patch
new file mode 100644
index 0000000..6b70bd4
--- /dev/null
+++ b/dev-db/scylla/files/0001-repair-Repair-tables-in-parallel.patch
@@ -0,0 +1,44 @@
+From 71b9eaaf6ec2c22715b772315321c28cdcfecdae Mon Sep 17 00:00:00 2001
+Message-Id: <71b9eaaf6ec2c22715b772315321c28cdcfecdae.1574946436.git.asias@scylladb.com>
+From: Asias He <asias@scylladb.com>
+Date: Thu, 28 Nov 2019 20:41:25 +0800
+Subject: [PATCH scylla] repair: Repair tables in parallel
+To: scylladb-dev@googlegroups.com
+
+---
+ repair/repair.cc | 6 +++++-
+ 1 file changed, 5 insertions(+), 1 deletion(-)
+
+diff --git a/repair/repair.cc b/repair/repair.cc
+index c3e72854d6..3485605d4a 100644
+--- a/repair/repair.cc
++++ b/repair/repair.cc
+@@ -938,6 +938,8 @@ static future<> repair_cf_range(repair_info& ri,
+ });
+ }
+
++static thread_local semaphore tables_parallelism_semaphore(16);
++
+ // Repair a single local range, multiple column families.
+ // Comparable to RepairSession in Origin
+ static future<> repair_range(repair_info& ri, const dht::token_range& range) {
+@@ -956,13 +958,15 @@ static future<> repair_range(repair_info& ri, const dht::token_range& range) {
+ neighbors.swap(live_neighbors);
+ }
+ return ::service::get_local_migration_manager().sync_schema(ri.db.local(), neighbors).then([&neighbors, &ri, range, id] {
+- return do_for_each(ri.cfs.begin(), ri.cfs.end(), [&ri, &neighbors, range] (auto&& cf) {
++ return parallel_for_each(ri.cfs.begin(), ri.cfs.end(), [&ri, &neighbors, range] (auto&& cf) {
++ return with_semaphore(tables_parallelism_semaphore, 1, [&ri, &neighbors, &cf, range] {
+ ri._sub_ranges_nr++;
+ if (ri.row_level_repair()) {
+ return repair_cf_range_row_level(ri, cf, range, neighbors);
+ } else {
+ return repair_cf_range(ri, cf, range, neighbors);
+ }
++ });
+ });
+ });
+ });
+--
+2.23.0
+
diff --git a/dev-db/scylla/scylla-3.1.1-r2.ebuild b/dev-db/scylla/scylla-3.1.1-r2.ebuild
new file mode 100644
index 0000000..8ded546
--- /dev/null
+++ b/dev-db/scylla/scylla-3.1.1-r2.ebuild
@@ -0,0 +1,253 @@
+# Copyright 1999-2019 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=6
+
+MY_PV="${PV/_beta/}"
+EGIT_CLONE_TYPE="mirror"
+EGIT_COMMIT="scylla-${PV}"
+EGIT_REPO_URI="https://github.com/scylladb/scylla.git"
+inherit git-r3
+
+PYTHON_COMPAT=( python3_{4,5,6} )
+
+inherit autotools flag-o-matic linux-info python-r1 toolchain-funcs systemd user
+
+DESCRIPTION="NoSQL data store using the seastar framework, compatible with Apache Cassandra"
+HOMEPAGE="http://scylladb.com/"
+
+KEYWORDS="~amd64"
+LICENSE="AGPL-3"
+SLOT="0"
+IUSE="doc systemd"
+
+# NOTE:
+# if you want to debug using backtraces, enable the 'splitdebug' FEATURE:
+# https://wiki.gentoo.org/wiki/Project:Quality_Assurance/Backtraces
+#
+# then check out:
+# https://github.com/scylladb/scylla/wiki/How-to-resolve-backtrace
+
+RESTRICT="test"
+
+RDEPEND="
+ >=dev-libs/thrift-0.11.0
+ <dev-util/ragel-7.0
+ <sys-apps/hwloc-2.0.0
+ ~app-admin/scylla-jmx-${PV}
+ ~app-admin/scylla-tools-${PV}
+ >=virtual/jdk-1.8.0:*
+ app-arch/lz4
+ app-arch/snappy
+ app-arch/zstd
+ dev-cpp/antlr-cpp:3.5
+ dev-cpp/yaml-cpp
+ dev-java/antlr:3.5
+ ~dev-libs/boost-1.65.0[icu]
+ dev-libs/crypto++
+ dev-libs/jsoncpp
+ dev-libs/libaio
+ >=dev-libs/libfmt-3.2.1
+ dev-libs/libxml2
+ dev-libs/protobuf
+ dev-python/psutil[${PYTHON_USEDEP}]
+ dev-python/pyparsing[${PYTHON_USEDEP}]
+ dev-python/pystache[${PYTHON_USEDEP}]
+ dev-python/pyudev[${PYTHON_USEDEP}]
+ >=dev-python/pyyaml-5.1[${PYTHON_USEDEP}]
+ dev-python/requests[${PYTHON_USEDEP}]
+ dev-python/urwid[${PYTHON_USEDEP}]
+ dev-util/systemtap
+ net-libs/gnutls
+ net-misc/lksctp-tools
+ sys-apps/ethtool
+ sys-fs/xfsprogs
+ sys-libs/libunwind
+ sys-libs/zlib
+ sys-process/numactl
+ x11-libs/libpciaccess
+ systemd? ( sys-apps/systemd )
+"
+DEPEND="${RDEPEND}
+ >=sys-kernel/linux-headers-3.5
+ dev-util/ninja
+"
+
+# Discussion about kernel configuration:
+# https://groups.google.com/forum/#!topic/scylladb-dev/qJu2zrryv-s
+CONFIG_CHECK="~NUMA_BALANCING ~SYN_COOKIES ~TRANSPARENT_HUGEPAGE"
+ERROR_NUMA_BALANCING="${PN} recommends support for Memory placement aware NUMA scheduler (NUMA_BALANCING)."
+ERROR_SYN_COOKIES="${PN} recommends support for TCP syncookie (SYN_COOKIES)."
+ERROR_TRANSPARENT_HUGEPAGE="${PN} recommends support for Transparent Hugepage (TRANSPARENT_HUGEPAGE)."
+
+# NOTE: maybe later depending on upstream energy, support DPDK
+# For DPDK, removed HUGETLBFS PROC_PAGE_MONITOR UIO_PCI_GENERIC in favor of VFIO
+# CONFIG_CHECK="~NUMA_BALANCING ~SYN_COOKIES ~TRANSPARENT_HUGEPAGE ~VFIO"
+# ERROR_VFIO="${PN} running with DPDK recommends support for Non-Privileged userspace driver framework (VFIO)."
+
+DOCS=( LICENSE.AGPL NOTICE.txt ORIGIN README.md README-DPDK.md )
+PATCHES=(
+ "${FILESDIR}/fix-fmt-3.5.0-compilation.patch"
+ "${FILESDIR}/3.1-thrift-support.patch"
+ "${FILESDIR}/v1-0001-repair-Add-a-timer-for-repair-reader.patch"
+ "${FILESDIR}/0001-repair-Repair-tables-in-parallel.patch"
+)
+
+pkg_setup() {
+ linux-info_pkg_setup
+ enewgroup scylla
+ enewuser scylla -1 -1 /var/lib/${PN} scylla
+}
+
+src_prepare() {
+ default
+
+ # set version
+ local git_commit=$(git log --pretty=format:'%h' -n 1)
+ echo "${PV}-${git_commit}" > version
+
+ # fix jsoncpp detection
+ sed -e 's@json/json.h@jsoncpp/json/json.h@g' -i json.hh || die
+
+ # QA: no -Werror (tho it does not propagate)
+ sed -e 's/\-Werror//g' -i configure.py || die
+
+ # QA NOTE: respect -O3 as it is upstream recommended
+ # and because we would kill the RAM of the machine with lower optimization
+ # since some files can take up to 8GB of RAM to compile!
+ # sed -e 's/\-O3//g' -i configure.py || die
+
+ # I don't agree with the old 4GB of RAM per job, it's more about 8GB now
+ sed -e 's/4000000000/8000000000/g' -i scripts/jobs || die
+
+ # TODO: contribute me
+ # fix pyyaml vulnerability #659348
+ # seastar/scripts/perftune.py
+ # dist/docker/redhat/scyllasetup.py
+ # dist/common/scripts/scylla_io_setup
+ # dist/common/scripts/scylla_fstrim
+ # dist/common/scripts/scylla_blocktune.py
+ # dist/common/scripts/scylla_util.py
+ # dist/common/scripts/scylla_config_get.py
+ find "${S}/dist" -type f -exec sed -e 's/yaml.load(/yaml.full_load(/g' -i {} \+ || die
+ sed -e 's/yaml.load(/yaml.safe_load(/g' -i seastar/scripts/perftune.py || die
+}
+
+src_configure() {
+ python_setup
+
+ # copied from dist/redhat/scylla.spec.mustache
+ # we want a package compiled with old kernel headers to
+ # support nowait aio if the user upgrades their kernel
+ if ! grep -qwr RWF_NOWAIT /usr/include/linux; then
+ append-cflags "-DRWF_NOWAIT=8"
+ fi
+ if ! grep -qwr aio_rw_flags /usr/include/linux; then
+ append-cflags "-Daio_rw_flags=aio_reserved1"
+ fi
+
+ # native CPU CFLAGS are strongly enforced by upstreams, respect that
+ replace-cpu-flags "*" "native"
+
+ append-cflags "-Wno-attributes -Wno-array-bounds"
+
+ ${EPYTHON} configure.py --enable-gcc6-concepts --mode=release --with=scylla --disable-xen --c-compiler "$(tc-getCC)" --compiler "$(tc-getCXX)" --ldflags "${LDFLAGS}" --cflags "${CFLAGS}" --python ${EPYTHON} --with-antlr3 /usr/bin/antlr3.5 || die
+}
+
+src_compile() {
+ # we use the provided 'scripts/jobs' to figure out how many parallel
+ # compilation jobs we can sustain
+ einfo "Compiling using $(scripts/jobs) jobs"
+ ninja -v build/release/scylla build/release/iotune -j$(scripts/jobs) || die
+}
+
+src_install() {
+ # NOTE:
+ # I base myself on upstream's install.sh file
+
+ default
+
+ local MUSTACHE_DIST="\"debian\": true"
+ pystache dist/common/systemd/scylla-server.service.mustache "{ $MUSTACHE_DIST }" > build/scylla-server.service
+ pystache dist/common/systemd/scylla-housekeeping-daily.service.mustache "{ $MUSTACHE_DIST }" > build/scylla-housekeeping-daily.service
+ pystache dist/common/systemd/scylla-housekeeping-restart.service.mustache "{ $MUSTACHE_DIST }" > build/scylla-housekeeping-restart.service
+
+ insinto /etc/default
+ doins dist/common/sysconfig/scylla-server
+
+ insinto /etc/security/limits.d
+ doins dist/common/limits.d/scylla.conf
+
+ insinto /etc/scylla.d
+ doins dist/common/scylla.d/*.conf
+
+ insinto /etc/sysctl.d
+ doins dist/common/sysctl.d/*.conf
+
+ insinto /etc/scylla
+ for config_file in scylla.yaml cassandra-rackdc.properties; do
+ doins conf/${config_file}
+ done
+
+ systemd_dounit build/*.service
+ systemd_dounit dist/common/systemd/*.service
+ systemd_dounit dist/common/systemd/*.timer
+
+ exeinto /usr/lib/scylla
+ doexe dist/common/scripts/*
+ doexe seastar/scripts/*
+ doexe scylla-gdb.py
+
+ dobin build/release/iotune
+ dobin build/release/scylla
+
+ insinto /usr/lib/scylla
+ doins dist/common/scripts/scylla_blocktune.py
+
+ exeinto /usr/lib/scylla
+ doexe dist/common/scripts/scylla-blocktune
+ doexe dist/common/scripts/scylla-housekeeping
+
+ insinto /etc/scylla.d
+ doins conf/housekeeping.cfg
+
+ newinitd "${FILESDIR}/scylla-server.initd" ${PN}-server
+ newconfd "${FILESDIR}/scylla-server.confd" ${PN}-server
+
+ dodoc -r licenses
+
+ for x in /var/lib/${PN} /var/lib/${PN}/{data,commitlog,hints,coredump} /var/lib/scylla-housekeeping /var/log/scylla; do
+ keepdir "${x}"
+ fowners scylla:scylla "${x}"
+ done
+
+ insinto /usr/lib/scylla/swagger-ui
+ doins -r swagger-ui/dist
+
+ insinto /usr/lib/scylla/api
+ doins -r api/api-doc
+
+ insinto /usr/lib/scylla/scyllatop
+ doins -r tools/scyllatop/*
+ fperms +x /usr/lib/scylla/scyllatop/scyllatop.py
+ dosym /usr/lib/scylla/scyllatop/scyllatop.py /usr/sbin/scyllatop
+
+ for util in $(ls dist/common/sbin/); do
+ dosym /usr/lib/scylla/${util} /usr/sbin/${util}
+ done
+
+ insinto /etc/sudoers.d
+ newins "${FILESDIR}"/scylla.sudoers scylla
+
+ insinto /etc/rsyslog.d
+ doins "${FILESDIR}/10-scylla.conf"
+}
+
+pkg_postinst() {
+ elog "You should run 'emerge --config dev-db/scylla' to finalize your ScyllaDB installation."
+}
+
+pkg_config() {
+ elog "Running 'scylla_setup'..."
+ scylla_setup
+}