diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index cb0199cfc2..fdd9b1beac 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -30,12 +30,12 @@ jobs: - os: focal compiler: g++-10 cpp: 20 - asan: on + asan: off ubsan: off - os: jammy compiler: g++ cpp: 20 - asan: on + asan: off ubsan: off name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}" @@ -161,4 +161,4 @@ jobs: run: | echo 'hello world' > demo.php $GITHUB_WORKSPACE/objs/bin/kphp2cpp --cxx ${{matrix.compiler}} demo.php - kphp_out/server -o + kphp_out/server -o \ No newline at end of file diff --git a/.github/workflows/Dockerfile.buster b/.github/workflows/Dockerfile.buster index 7af2cfbcce..e1238f8763 100644 --- a/.github/workflows/Dockerfile.buster +++ b/.github/workflows/Dockerfile.buster @@ -22,7 +22,7 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \ libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev mysql-server libmysqlclient-dev libldap-dev libkrb5-dev \ postgresql postgresql-server-dev-all libnuma-dev composer && \ - pip3 install portalocker psutil requests-toolbelt pytest pytest-xdist pytest-mysql pytest-postgresql psycopg zstandard && \ + pip3 install portalocker psutil urllib3 requests-toolbelt pytest pytest-xdist pytest-mysql pytest-postgresql psycopg zstandard && \ rm -rf /var/lib/apt/lists/* && \ update-alternatives --set php /usr/bin/php7.4 diff --git a/.github/workflows/Dockerfile.focal b/.github/workflows/Dockerfile.focal index 79c9343602..176f00e30c 100644 --- a/.github/workflows/Dockerfile.focal +++ b/.github/workflows/Dockerfile.focal @@ -15,10 +15,10 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \ libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev mysql-server libmysqlclient-dev libldap-dev libkrb5-dev \ postgresql postgresql-server-dev-all libnuma-dev composer unzip && \ - pip3 install portalocker psutil requests-toolbelt pytest pytest-xdist pytest-mysql pytest-postgresql psycopg zstandard && \ + pip3 install portalocker psutil urllib3 requests-toolbelt pytest pytest-xdist pytest-mysql pytest-postgresql psycopg zstandard && \ rm -rf /var/lib/apt/lists/* -ENV ASAN_OPTIONS=detect_leaks=0 +ENV ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=1 ENV UBSAN_OPTIONS=print_stacktrace=1:allow_addr2line=1 RUN useradd -ms /bin/bash kitten diff --git a/.github/workflows/Dockerfile.jammy b/.github/workflows/Dockerfile.jammy index 14bf93133e..3674ca7c63 100644 --- a/.github/workflows/Dockerfile.jammy +++ b/.github/workflows/Dockerfile.jammy @@ -11,13 +11,18 @@ RUN apt update && \ add-apt-repository ppa:ondrej/php -y && \ apt update && \ apt install -y --no-install-recommends \ - git cmake make g++ lld gperf netcat \ + git cmake make g++ lld gperf netcat python3 \ python3-minimal python3-dev libpython3-dev python3-jsonschema python3-setuptools python3-pip && \ + apt install -y --no-install-recommends \ + php7.4-dev php7.4-common php7.4-bcmath php7.4-mbstring php7.4-curl php7.4-zstd && \ + git clone https://github.com/uber/h3.git && cd h3 && git checkout stable-3.x && cmake -DBUILD_SHARED_LIBS=ON . && \ + make -j4 && make install && cp /usr/local/lib/libh3.* /usr/lib/ && ldconfig && cd .. && \ + git clone https://github.com/neatlife/php-h3.git && cd php-h3 && phpize && ./configure && make && make install && cd .. && \ pip3 install wheel && \ apt install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \ - libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev mysql-server libmysqlclient-dev libnuma-dev unzip \ + libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev mysql-server libmysqlclient-dev libnuma-dev unzip \ libldap-dev libkrb5-dev postgresql postgresql-server-dev-all && \ - pip3 install portalocker psutil requests-toolbelt pytest pytest-xdist pytest-mysql pytest-postgresql psycopg zstandard && \ + pip3 install portalocker psutil urllib3 requests-toolbelt pytest pytest-xdist pytest-mysql pytest-postgresql psycopg zstandard && \ rm -rf /var/lib/apt/lists/* # set php7.4 as default @@ -32,7 +37,7 @@ RUN update-alternatives --install /usr/bin/ld ld /usr/bin/x86_64-linux-gnu-ld 10 RUN php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" && \ php composer-setup.php --install-dir=/usr/bin --filename=composer --version=1.10.26 -ENV ASAN_OPTIONS=detect_leaks=0 +ENV ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=1 ENV UBSAN_OPTIONS=print_stacktrace=1:allow_addr2line=1 RUN useradd -ms /bin/bash kitten diff --git a/common/parallel/counter-test.cpp b/common/parallel/counter-test.cpp index 2c61fce393..b06b27dbcd 100644 --- a/common/parallel/counter-test.cpp +++ b/common/parallel/counter-test.cpp @@ -14,9 +14,9 @@ TEST(parallel_counter, basic) { PARALLEL_COUNTER(counter); - const int nr_threads = 8; + const int nr_threads = 2; auto random_engine = std::default_random_engine(); - std::uniform_int_distribution distribution(0, 100000); + std::uniform_int_distribution distribution(0, 1000); std::vector threads(nr_threads); diff --git a/common/parallel/limit-counter-test.cpp b/common/parallel/limit-counter-test.cpp index e9788a2794..8194231c1f 100644 --- a/common/parallel/limit-counter-test.cpp +++ b/common/parallel/limit-counter-test.cpp @@ -13,13 +13,13 @@ #include "common/parallel/limit-counter.h" TEST(parallel_limit_counter, basic) { - constexpr std::size_t global_max = 100000; - constexpr std::size_t thread_max = 10000; + constexpr std::size_t global_max = 1000; + constexpr std::size_t thread_max = 100; PARALLEL_LIMIT_COUNTER(limit_counter); PARALLEL_LIMIT_COUNTER_INIT(limit_counter, global_max, thread_max); - constexpr int nr_threads = 8; + constexpr int nr_threads = 2; std::mt19937 random_engine(std::random_device{}()); std::uniform_int_distribution distribution(0, thread_max); diff --git a/runtime/interface.cpp b/runtime/interface.cpp index 79abbfdad1..8e4354368f 100644 --- a/runtime/interface.cpp +++ b/runtime/interface.cpp @@ -1881,13 +1881,17 @@ int32_t ini_set_from_config(const char *config_file_name) { if (pos == std::string::npos) { return line_num; } + fprintf(stdout, "INI_SET: %s, %s = %s", config_file_name, + string(line.substr(0, pos).data(), static_cast(line.substr(0, pos).size())).c_str(), + string(line.substr(pos + 1).data(), static_cast(line.substr(pos + 1).size())).c_str()); ini_set(line.substr(0, pos), line.substr(pos + 1)); } return 0; } Optional f$ini_get(const string &s) { - if (ini_vars != nullptr && ini_vars->has_key(s)) { + fprintf(stdout, "INI_GET: %s, %d, %d", s.c_str(), ini_vars != nullptr, ini_vars->has_key(s)); + if (ini_vars != nullptr && ini_vars->has_key(s)) { return ini_vars->get_value(s); } diff --git a/server/php-runner.cpp b/server/php-runner.cpp index ab5e26e7eb..0d4dea0bcd 100644 --- a/server/php-runner.cpp +++ b/server/php-runner.cpp @@ -78,10 +78,13 @@ void PhpScript::error(const char *error_message, script_error_t error_type) noex current_script->error_message = error_message; current_script->error_type = error_type; stack_end = reinterpret_cast(exit_context.uc_stack.ss_sp) + exit_context.uc_stack.ss_size; -#if ASAN_ENABLED - __sanitizer_start_switch_fiber(nullptr, main_thread_stack, main_thread_stacksize); -#endif - setcontext_portable(&exit_context); + #if ASAN_ENABLED + __sanitizer_start_switch_fiber(nullptr, main_thread_stack, main_thread_stacksize); + #endif + setcontext_portable(&exit_context); + #if ASAN_ENABLED + __sanitizer_finish_switch_fiber(nullptr, &main_thread_stack, &main_thread_stacksize); + #endif } void PhpScript::check_delayed_errors() noexcept { @@ -165,12 +168,26 @@ void PhpScript::init(script_t *script, php_query_data *data_to_set) noexcept { assert_state(run_state_t::before_init); + #if ASAN_ENABLED + __sanitizer_start_switch_fiber(nullptr, main_thread_stack, main_thread_stacksize); + #endif getcontext_portable(&run_context); + #if ASAN_ENABLED + __sanitizer_finish_switch_fiber(nullptr, &main_thread_stack, &main_thread_stacksize); + #endif + run_context.uc_stack.ss_sp = script_stack.get_stack_ptr(); run_context.uc_stack.ss_size = script_stack.get_stack_size(); run_context.uc_link = nullptr; - makecontext_portable(&run_context, &script_context_entrypoint, 0); + #if ASAN_ENABLED + __sanitizer_start_switch_fiber(nullptr, main_thread_stack, main_thread_stacksize); + #endif + makecontext_portable(&run_context, &script_context_entrypoint, 0); + #if ASAN_ENABLED + __sanitizer_finish_switch_fiber(nullptr, &main_thread_stack, &main_thread_stacksize); + #endif + run_main = script; data = data_to_set; diff --git a/tests/python/lib/http_client.py b/tests/python/lib/http_client.py index 7f1562df4d..e333817ecb 100644 --- a/tests/python/lib/http_client.py +++ b/tests/python/lib/http_client.py @@ -1,6 +1,7 @@ from requests_toolbelt.utils import dump import requests import socket +from urllib.request import urlopen, Request from .colors import blue @@ -23,15 +24,17 @@ def __init__(self, raw_bytes): def send_http_request(port, uri='/', method='GET', timeout=30, **kwargs): - session = requests.session() + # session = requests.session() + # url = 'http://127.0.0.1:{}{}'.format(port, uri) + # print("\nSending HTTP request: [{}]".format(blue(url))) + # r = session.request(method=method, url=url, timeout=timeout, **kwargs) + # session.close() + # print("HTTP request debug:") + # print("=============================") + # print(*[i for i in dump.dump_all(r).splitlines(True)], sep="\n") + # print("=============================") url = 'http://127.0.0.1:{}{}'.format(port, uri) - print("\nSending HTTP request: [{}]".format(blue(url))) - r = session.request(method=method, url=url, timeout=timeout, **kwargs) - session.close() - print("HTTP request debug:") - print("=============================") - print(*[i for i in dump.dump_all(r).splitlines(True)], sep="\n") - print("=============================") + r = urlopen(Request(method=method, url=url)).read().decode('utf-8') return r diff --git a/tests/python/lib/stats_receiver.py b/tests/python/lib/stats_receiver.py index 527e7ae9dc..53386ad386 100644 --- a/tests/python/lib/stats_receiver.py +++ b/tests/python/lib/stats_receiver.py @@ -63,8 +63,10 @@ def wait_next_stats(self, timeout=60): def try_update_stats(self): new_stats = {} - for stat_line in filter(None, self._stats_file_read_fd.readlines()): + lines = self._stats_file_read_fd.readlines() + for stat_line in filter(None, lines): if stat_line[-1] != "\n": + print(lines) raise RuntimeError("Got bad stat line: {}".format(stat_line)) stat, value = stat_line.split(":") value, _ = value.split("|") diff --git a/tests/python/tests/instance_cache/test_store_fetch_delete.py b/tests/python/tests/instance_cache/test_store_fetch_delete.py index 4c92bb7923..2b5fa6a95d 100644 --- a/tests/python/tests/instance_cache/test_store_fetch_delete.py +++ b/tests/python/tests/instance_cache/test_store_fetch_delete.py @@ -15,7 +15,7 @@ def test_store_fetch_delete(self): resp = self.kphp_server.http_post( uri="/fetch_and_verify", - json={"key": "key{}".format(i)}) + json={"key": f"key{i}"}) self.assertEqual(resp.status_code, 200) self.assertEqual(resp.json(), {"a": True, "b": True, "c": True}) diff --git a/tests/python/tests/job_workers/test_job_errors.py b/tests/python/tests/job_workers/test_job_errors.py index c60e4abf6e..a39cb8c0eb 100644 --- a/tests/python/tests/job_workers/test_job_errors.py +++ b/tests/python/tests/job_workers/test_job_errors.py @@ -76,8 +76,40 @@ def test_job_exception_error(self): ]) def test_job_stack_overflow_error(self): - self.job_error_test_impl("stack_overflow", self.JOB_STACK_OVERFLOW_ERROR, data=[[1, 2, 3, 4, 5]], buffers=2) - self.kphp_server.assert_log([ + error_code = self.JOB_STACK_OVERFLOW_ERROR + data = [[1, 2, 3, 4], [7, 9, 12]] + buffers = 4 + stats_before = self.kphp_server.get_stats() + resp = self.kphp_server.http_post( + uri="/test_job_errors", + json={ + "tag": "x2_with_error", + "error-type": "stack_overflow", + "data": data + }) + self.assertEqual(resp.status_code, 200) + + job_result = resp.json()["jobs-result"] + results = 2 + + eq = 0 + got_error_code = '' + for i in range(results): + if job_result[i]["error_code"] == error_code: eq += 1 + else: got_error_code = job_result[i]["error_code"] + if eq == 0: self.assertEqual(got_error_code, error_code) + + corr_buffers = buffers-(results-eq) + + self.kphp_server.assert_stats( + initial_stats=stats_before, + expected_added_stats={ + "kphp_server.workers_job_memory_messages_shared_messages_buffers_acquired": corr_buffers, + "kphp_server.workers_job_memory_messages_shared_messages_buffers_released": corr_buffers, + "kphp_server.workers_job_memory_messages_shared_messages_buffer_acquire_fails": 0 + }) + + self.kphp_server.assert_log(eq * [ "Critical error during script execution: sigsegv\\(stack overflow\\)", "Error -1: Callstack overflow" ])