diff --git a/.github/workflows/fast_testing.yml b/.github/workflows/fast_testing.yml deleted file mode 100644 index 64dea183..00000000 --- a/.github/workflows/fast_testing.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: fast_testing - -on: - workflow_dispatch: - pull_request: - push: - branches: - - 'master' - tags: - - '*' - -jobs: - run_tests: - runs-on: ubuntu-22.04 - - strategy: - fail-fast: false - matrix: - tarantool: - - '1.10' - - '2.11' - - '3.4' - - steps: - - name: Clone the module - uses: actions/checkout@v3 - - - name: Setup tarantool ${{ matrix.tarantool }} - uses: tarantool/setup-tarantool@v3 - with: - tarantool-version: ${{ matrix.tarantool }} - - - run: cmake . && make check diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml deleted file mode 100644 index 0d04670d..00000000 --- a/.github/workflows/packaging.yml +++ /dev/null @@ -1,111 +0,0 @@ -name: packaging - -on: [push, pull_request] - -jobs: - # Run not only on tags, otherwise dependent job will skip. - version-check: - # Skip pull request job when the source branch is in the same - # repository. - if: | - github.event_name == 'push' || - github.event.pull_request.head.repo.full_name != github.repository - runs-on: ubuntu-24.04 - steps: - - name: Check module version - # We need this step to run only on push with tag. - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }} - uses: tarantool/actions/check-module-version@master - with: - module-name: 'queue' - - package: - # Skip pull request jobs when the source branch is in the same - # repository. - if: | - github.event_name == 'push' || - github.event.pull_request.head.repo.full_name != github.repository - # Packaging for CentOS 7 does not work with other versions, see: - # https://github.com/packpack/packpack/issues/145 - runs-on: ubuntu-24.04 - needs: version-check - - strategy: - fail-fast: false - matrix: - platform: - - { os: 'debian', dist: 'stretch' } - - { os: 'debian', dist: 'buster' } - - { os: 'debian', dist: 'bullseye' } - - { os: 'el', dist: '7' } - - { os: 'el', dist: '8' } - - { os: 'fedora', dist: '30' } - - { os: 'fedora', dist: '31' } - - { os: 'fedora', dist: '32' } - - { os: 'fedora', dist: '33' } - - { os: 'fedora', dist: '34' } - - { os: 'fedora', dist: '35' } - - { os: 'fedora', dist: '36' } - - { os: 'ubuntu', dist: 'xenial' } - - { os: 'ubuntu', dist: 'bionic' } - - { os: 'ubuntu', dist: 'focal' } - - { os: 'ubuntu', dist: 'groovy' } - - { os: 'ubuntu', dist: 'jammy' } - - env: - OS: ${{ matrix.platform.os }} - DIST: ${{ matrix.platform.dist }} - - steps: - - name: Clone the module - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Clone the packpack tool - uses: actions/checkout@v2 - with: - repository: packpack/packpack - path: packpack - - - name: Fetch tags - # Found that Github checkout Actions pulls all the tags, but - # right it deannotates the testing tag, check: - # https://github.com/actions/checkout/issues/290 - # But we use 'git describe ..' calls w/o '--tags' flag and it - # prevents us from getting the needed tag for packages version - # setup. To avoid of it, let's fetch it manually, to be sure - # that all tags will exists always. - run: git fetch --tags -f - - - name: Create packages - run: ./packpack/packpack - - - name: Deploy packages - # We need this step to run only on push with tag. - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }} - env: - RWS_URL_PART: https://rws.tarantool.org/tarantool-modules - RWS_AUTH: ${{ secrets.RWS_AUTH }} - PRODUCT_NAME: tarantool-queue - run: | - CURL_CMD="curl -LfsS \ - -X PUT ${RWS_URL_PART}/${OS}/${DIST} \ - -u ${RWS_AUTH} \ - -F product=${PRODUCT_NAME}" - - # We don't want to try to print secrets to the log, but we want - # to print a "curl" command to see what's going on. - CURL_CMD_ECHO="curl -LfsS \ - -X PUT ${RWS_URL_PART}/${OS}/${DIST} \ - -u '***' \ - -F product=${PRODUCT_NAME}" - - for f in $(ls -I '*build*' -I '*.changes' ./build); do - CURL_CMD+=" -F $(basename ${f})=@./build/${f}" - CURL_CMD_ECHO+=" -F $(basename ${f})=@./build/${f}" - done - - echo ${CURL_CMD_ECHO} - - ${CURL_CMD} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 744b4402..00000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: publish - -on: - push: - branches: [master] - tags: ['*'] - -jobs: - version-check: - # We need this job to run only on push with tag. - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }} - runs-on: ubuntu-24.04 - steps: - - name: Check module version - uses: tarantool/actions/check-module-version@master - with: - module-name: 'queue' - - publish-rockspec-scm-1: - if: github.ref == 'refs/heads/master' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: tarantool/rocks.tarantool.org/github-action@master - with: - auth: ${{ secrets.ROCKS_AUTH }} - files: queue-scm-1.rockspec - - publish-rockspec-tag: - if: startsWith(github.ref, 'refs/tags/') - runs-on: ubuntu-22.04 - needs: version-check - steps: - - uses: actions/checkout@v3 - - # Create a rockspec for the release. - - run: printf '%s=%s\n' TAG "${GITHUB_REF##*/}" >> "${GITHUB_ENV}" - - run: sed -E - -e "s/branch = '.+'/tag = '${{ env.TAG }}'/g" - -e "s/version = '.+'/version = '${{ env.TAG }}-1'/g" - queue-scm-1.rockspec > queue-${{ env.TAG }}-1.rockspec - - # Create a rock for the release (.all.rock). - # - # `tarantoolctl rocks pack ` creates - # .all.rock tarball. It speeds up - # `tarantoolctl rocks install ` and - # frees it from dependency on git. - # - # Don't confuse this command with - # `tarantoolctl rocks pack `, which creates a - # source tarball (.src.rock). - # - # Important: Don't upload binary rocks to - # rocks.tarantool.org. Lua/C modules should be packed into - # .src.rock instead. See [1] for description of rock types. - # - # [1]: https://github.com/luarocks/luarocks/wiki/Types-of-rocks - - uses: tarantool/setup-tarantool@v3 - with: - tarantool-version: '2.10' - - run: tarantoolctl rocks install queue-${{ env.TAG }}-1.rockspec - - run: tarantoolctl rocks pack queue ${{ env.TAG }} - - # Upload .rockspec and .all.rock. - - uses: tarantool/rocks.tarantool.org/github-action@master - with: - auth: ${{ secrets.ROCKS_AUTH }} - files: | - queue-${{ env.TAG }}-1.rockspec - queue-${{ env.TAG }}-1.all.rock diff --git a/.github/workflows/reusable_testing.yml b/.github/workflows/reusable_testing.yml deleted file mode 100644 index ffec90b1..00000000 --- a/.github/workflows/reusable_testing.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: reusable_testing - -on: - workflow_call: - inputs: - artifact_name: - description: 'The name of the tarantool build artifact' - default: ubuntu-focal - required: false - type: string - -jobs: - run_tests: - runs-on: ubuntu-24.04 - steps: - - name: 'Clone the queue module' - uses: actions/checkout@v4 - with: - repository: ${{ github.repository_owner }}/queue - - - name: 'Download the tarantool build artifact' - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.artifact_name }} - - - name: 'Install tarantool' - # Now we're lucky: all dependencies are already installed. Check package - # dependencies when migrating to other OS version. - run: sudo dpkg -i tarantool*.deb - - - run: cmake . && make check diff --git a/.gitignore b/.gitignore index e991bd40..3a4edf69 100644 --- a/.gitignore +++ b/.gitignore @@ -1,18 +1 @@ -t/var -VERSION -# cmake -CMakeFiles -cmake_install.cmake -install_manifest.txt -CMakeCache.txt -Makefile -# ctest -Testing -CTestTestfile.cmake -# tarantool garbage -*.snap -*.xlog -*.vylog -# vim -*~ -.*.swp +.project diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..6d83f266 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,14 @@ +language: perl +perl: + - "5.18" + +before_script: + - uname -a + - lsb_release -c -s + - curl http://tarantool.org/dist/public.key | sudo apt-key add - + - echo "deb http://tarantool.org/dist/stable/ubuntu/ `lsb_release -c -s` main" | sudo tee -a /etc/apt/sources.list.d/tarantool.list + - sudo apt-get update > /dev/null + - sudo apt-get -q install tarantool-lts tarantool-lts-dev + +script: + - perl Makefile.PL && make test diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 07ac4d4e..00000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,254 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) -and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). - -## [Unreleased] - -### Added - -### Changed - -### Fixed - -## [1.4.4] - 2025-05-26 - -The patch release fixes incorrect behavior of the utubettl driver with enabled -`STORAGE_MODE_READY_BUFFER` mode. - -### Fixed - -- Incorrect choice by priority of task with utubettl driver + ready buffer - mode (#244). -- Unable to take task with utubettl driver + ready buffer mode (#244). - -## [1.4.3] - 2025-03-05 - -The release fixes start of a queue on instances with gaps inside -the box.info.replication array. - -### Fixed - -- Grant method was added for `*_ready_buffer` spaces (#237). -- Attempt to index a nil value if box.info.replication array has gaps. - -## [1.4.2] - 2024-08-10 - -The release re-publish packages. - -### Fixed - -- Package publishing. - -## [1.4.1] - 2024-05-21 - -The release fixes too long release time by a `ttr` value or a `delay` timeout. - -### Fixed - -- Too long timings in the `utubettl` driver (#223). - -## [1.4.0] - 2024-05-20 - -The release introduces an experimental `storage_mode` option for the `utube` -and `utubettl` drivers with the `memtx` engine. It could be used to create a -`utube` or `utubettl` queue with an additional buffer space that stores only -ready to take tasks. - -```Lua -local tube = queue.create_tube('utube_with_ready_buffer', 'utube', - {storage_mode = queue.driver.utube.STORAGE_MODE_READY_BUFFER}) -``` -```Lua -local tube = queue.create_tube('utubettl_with_ready_buffer', 'utubettl', - {storage_mode = queue.driver.utubettl.STORAGE_MODE_READY_BUFFER}) -``` - -The storage mode slower in general cases, but a much faster in cases when -you have utubes with many tasks (see README.md for the performance comparison). -So you should make your choice carefully. - -### Added -- Experimental `storage_mode` option for creating a `utube` and `utubettl` - tube (#228). - It enables the workaround for slow takes while working with busy tubes. - -### Fixed - -- Stuck in `INIT` state if an instance failed to enter the `running` mode - in time (#226). This fix works only for Tarantool versions >= 2.10.0. -- Slow takes on busy `utube` and `utubettl` tubes (#228). The workaround could - be enabled by passing the `storage_mode = "ready_buffer"` option while - creating the tube. - -## [1.3.3] - 2023-09-13 - -### Fixed - -- In replicaset mode, the behavior of the public API is reduced to the same behavior - in all queue states, including INIT. Previously, in the INIT state, an ambiguous - error was thrown when trying to access a public method on a replica and the script - was interrupted by an error. - - Old behavior (call `create_tube` on replica, queue is in INIT state): - ``` - 2023-09-04 14:01:11.000 [5990] main/103/replica.lua/box.load_cfg I> set 'read_only' configuration option to true - stack traceback: - /home/void/tmp/cluster/repl/queue/init.lua:44: in function '__index' - replica.lua:13: in main chunk - 2023-09-04 14:01:11.004 [5990] main/105/checkpoint_daemon I> scheduled next checkpoint for Mon Sep 4 15:11:32 2023 - 2023-09-04 14:01:11.004 [5990] main utils.c:610 E> LuajitError: /home/void/tmp/cluster/repl/queue/init.lua:45: Please configure box.cfg{} in read/write mode first - ``` - After this fix: - ``` - 2023-09-11 10:24:31.463 [19773] main/103/replica.lua abstract.lua:93 E> create_tube: queue is in INIT state - ``` - -## [1.3.2] - 2023-08-24 - -### Fixed - -- Duplicate id error with mvvc on put and take (#207). - -## [1.3.1] - 2023-07-31 - -### Fixed - -- Yield in the fifottl/utubettl queue drivers. - -## [1.3.0] - 2023-03-13 - -### Added - -- Possibility to get the module version. - -### Fixed - -- Bug when working with the replicaset (#202). - -## [1.2.5] - 2023-02-28 - -This is a technical release that should fix several problems in the -repositories with packages that were caused by the mistaken creation -of several tags (for this reason there are no 1.2.3 and 1.2.4 tags -and corresponding packages in the repositories). - -## [1.2.2] - 2022-11-03 - -### Fixed - -- Excessive CPU consumption of the state fiber. - -## [1.2.1] - 2022-09-12 - -### Added - -- Rockspec publishing to CD. - -### Fixed - -- "replication" mode switching on tarantool < 2.2.1 - -## [1.2.0] - 2022-06-06 - -### Added - -- Master-replica switching support. - Added the ability to use a queue in the master replica scheme. - More information is available: - https://github.com/tarantool/queue#queue-state-diagram - https://github.com/tarantool/queue#queue-and-replication -- Granting privilege to `statistics`, `put` and `truncate`. - -### Fixed - -- Work with "ttl" of buried task. - Previously, if a task was "buried" after it was "taken" (and the task has a - "ttr"), then the time when the "release" should occur (the "ttr" timer) will - be interpreted as the end of the "ttl" timer and the task will be deleted. - -## [1.1.0] - 2020-12-25 - -### Added - -- "Shared sessions" was added to the queue. Previously, a connection to server - was synonym of the queue session. Now the session has a unique UUID (returned - by the "queue.identify()" method), and one session can have many connections. - To connect to an existing session, call "queue.identify(uuid)" with the - previously obtained UUID. -- Possibility to work with tasks after reconnect. The queue module now provides - the ability to set the `ttr` setting for sessions by - `queue.cfg({ ttr = ttr_in_sec})`, which characterizes how long the logical - session will exist after all active connections are closed. - -### Fixed - -- Custom driver registration after reboot. Previously, if a custom driver is - registered after calling box.cfg() it causes a problem when the instance will - be restarted. - -## [1.0.8] - 2020-10-17 - -### Added - -- The ability to start an instance with a loaded queue module in read-only - mode. In this case, a start of the module will be delayed until the instance - will be configured with read_only = false. Previously, when trying to - initialize the queue module on an instance configured in ro mode, an error - occurred: "ER_READONLY: Can't modify data because this instance is in - read-only mode." See https://github.com/tarantool/queue#initialization for - more details. - -## [1.0.7] - 2020-09-03 - -### Breaking changes - -- A check for a driver API implementation was added. Now, the consumer will be - informed about the missing methods in the driver implementation (an error will - be thrown). - -### Added - -- Notification about missing methods in the driver implementation (#126). - -### Fixed - -- The tasks releases on start for some drivers (utubettl, fifottl, - limfifottl). Before, an attempt to release "taken" tasks on start lead to an - error: "attempt to index local 'opts' (a nil value)" (#121). - -### Changed - -- Updated the requirements for the "delete" driver method. Now, the method - should change the state of a task to "done". Before, it was duplicated by - external code. - -## [1.0.6] - 2020-02-29 - -### Breaking changes - -External drivers should be updated with the new `tasks_by_state()` method. -Consider the example from the `fifo` driver: - -```lua --- get iterator to tasks in a certain state -function method.tasks_by_state(self, task_state) - return self.space.index.status:pairs(task_state) -end -``` - -The example uses 'status' secondary index, which is built on top of 'status' -and 'task_id' fields. - -This new method is necessary to correctly manage state of tasks: when tarantool -instance is restarted we should release all taken tasks (otherwise they would -stuck in the taken state forever). See #66 and #126 for more information. - -### Fixed - -- Releasing tasks at a client disconnection (#103). -- Optimize statistics build (-15% in some cases) (#92). -- Release all taken tasks at start (#66). -- Don't take tasks during a client disconnection (#104). diff --git a/CMakeLists.txt b/CMakeLists.txt deleted file mode 100644 index 613e7ac8..00000000 --- a/CMakeLists.txt +++ /dev/null @@ -1,31 +0,0 @@ -cmake_minimum_required(VERSION 2.8 FATAL_ERROR) - -project(queue) - -if(NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE RelWithDebInfo) -endif() -set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH}) - -set(TARANTOOL_FIND_REQUIRED ON) -find_package(Tarantool) - -add_subdirectory(queue) - -enable_testing() - -set (LUA_PATH "LUA_PATH=${PROJECT_SOURCE_DIR}/?.lua\\;${PROJECT_SOURCE_DIR}/?/init.lua\\;") - -file(GLOB TEST_LIST ${CMAKE_SOURCE_DIR}/t/*.t) - -foreach(TEST ${TEST_LIST}) - get_filename_component(TEST_NAME ${TEST} NAME) - add_test (memtx-${TEST_NAME} tarantool ${TEST}) - set_tests_properties(memtx-${TEST_NAME} PROPERTIES ENVIRONMENT "ENGINE=memtx;${LUA_PATH}") - add_test (vinyl-${TEST_NAME} tarantool ${TEST}) - set_tests_properties(vinyl-${TEST_NAME} PROPERTIES ENVIRONMENT "ENGINE=vinyl;${LUA_PATH}") -endforeach() - -add_custom_target(check - WORKING_DIRECTORY ${PROJECT_BUILD_DIR} - COMMAND ctest -V) diff --git a/LICENSE b/LICENSE deleted file mode 100644 index c2cd2b30..00000000 --- a/LICENSE +++ /dev/null @@ -1,31 +0,0 @@ -Copyright (C) 2014-2016 Tarantool AUTHORS: -please see AUTHORS file in tarantool/tarantool repository. - -/* - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * 1. Redistributions of source code must retain the above - * copyright notice, this list of conditions and the - * following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY AUTHORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED - * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL - * AUTHORS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, - * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF - * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ diff --git a/MANIFEST b/MANIFEST new file mode 100644 index 00000000..03b38f51 --- /dev/null +++ b/MANIFEST @@ -0,0 +1,30 @@ +benchmark/channel.pl +benchmark/init.lua +benchmark/ping.pl +benchmark/pqueue-mix.pl +benchmark/pqueue.pl +benchmark/queue.pl +debian/changelog +debian/compat +debian/control +debian/copyright +debian/dr-tarantool-queue.dirs +debian/dr-tarantool-queue.install +debian/libdr-tarantoolqueue-perl.docs +debian/libdr-tarantoolqueue-perl.install +debian/rules +debian/source/format +debian/watch +init.lua +lib/DR/TarantoolQueue.pm +lib/DR/TarantoolQueue/JSE.pm +lib/DR/TarantoolQueue/Task.pm +lib/DR/TarantoolQueue/Worker.pm +Makefile.PL +MANIFEST +README.md +t/000-queue.t +t/010-dr-tqueue.t +t/020-worker.t +t/030-parallel.t +tarantool.cfg diff --git a/Makefile.PL b/Makefile.PL new file mode 100644 index 00000000..cc2c8c88 --- /dev/null +++ b/Makefile.PL @@ -0,0 +1,25 @@ +use 5.008008; +use ExtUtils::MakeMaker; +WriteMakefile( + NAME => 'DR::TarantoolQueue', + VERSION_FROM => 'lib/DR/TarantoolQueue.pm', # finds $VERSION + PREREQ_PM => { + 'Carp' => 0, + 'Mouse' => 0, + 'DR::Tarantool' => 0, + 'Coro' => 0, + 'JSON::XS' => 0, + }, + META_MERGE => { + resources => { + homepage => 'https://github.com/mailru/tarantool-queue', + bugtracker => 'https://github.com/mailru/tarantool-queue/issues', + } + }, + ABSTRACT_FROM => 'lib/DR/TarantoolQueue.pm', + AUTHOR => 'Dmitry E. Oboukhov ', + LICENSE => 'perl', +); + +# open my $file, '>>', 'Makefile'; +# print $file "\n\nTEST_VERBOSE = 1\n"; diff --git a/README.md b/README.md index 95b98e5c..8231d55f 100644 --- a/README.md +++ b/README.md @@ -1,1003 +1,308 @@ - - - - -[![fast_testing][testing-actions-badge]][testing-actions-url] -[![packaging][packaging-actions-badge]][packaging-actions-url] -[![publish][publish-actions-badge]][publish-actions-url] - -# A collection of persistent queue implementations for Tarantool - -## Table of contents - -* [Queue types](#queue-types) - * [fifo \- a simple queue](#fifo---a-simple-queue) - * [fifottl \- a simple priority queue with support for task time to live](#fifottl---a-simple-priority-queue-with-support-for-task-time-to-live) - * [utube \- a queue with sub\-queues inside](#utube---a-queue-with-sub-queues-inside) - * [utubettl \- extension of utube to support ttl](#utubettl---extension-of-utube-to-support-ttl) -* [The underlying spaces](#the-underlying-spaces) - * [Fields of the \_queue space](#fields-of-the-_queue-space) - * [Fields of the \_queue\_consumers space](#fields-of-the-_queue_consumers-space) - * [Fields of the \_queue\_taken\_2 space](#fields-of-the-_queue_taken_2-space) - * [Fields of the \_queue\_session\_ids space](#fields-of-the-_queue_session_ids-space) - * [Fields of the space associated with each queue](#fields-of-the-space-associated-with-each-queue) -* [Task state diagram](#task-state-diagram) -* [Queue state diagram](#queue-state-diagram) -* [Installing](#installing) -* [Using the queue module](#using-the-queue-module) - * [Initialization](#initialization) - * [Get the module version](#get-the-module-version) - * [Creating a new queue](#creating-a-new-queue) - * [Set queue settings](#set-queue-settings) - * [Session identify](#session-identify) - * [Putting a task in a queue](#putting-a-task-in-a-queue) - * [Taking a task from the queue ("consuming")](#taking-a-task-from-the-queue-consuming) - * [Acknowledging the completion of a task](#acknowledging-the-completion-of-a-task) - * [Releasing a task](#releasing-a-task) - * [Peeking at a task](#peeking-at-a-task) - * [Burying a task](#burying-a-task) - * [Kicking a number of tasks](#kicking-a-number-of-tasks) - * [Deleting a task](#deleting-a-task) - * [Dropping a queue](#dropping-a-queue) - * [Releasing all taken tasks](#releasing-all-taken-tasks) - * [Getting statistics](#getting-statistics) - * [Queue and replication](#queue-and-replication) -* [Implementation details](#implementation-details) - * [Queue drivers](#queue-drivers) - * [Driver API](#driver-api) - -# Queue types - -## `fifo` - a simple queue - -Features: - -* If there is only one consumer, tasks are scheduled in strict FIFO order. -* If there are many concurrent consumers, FIFO order is preserved on -average, but is less strict: concurrent consumers may complete tasks in -a different order. - -The following options can be specified when creating a `fifo` queue: - * `temporary` - boolean - if true, the contents do not persist on disk -(the queue is in-memory only) - * `if_not_exists` - boolean - if true, no error will be returned if the tube -already exists - * `on_task_change` - function name - a callback to be executed on every -operation; the expected function syntax is `function(task, stats_data)`, where -`stats_data` is the operation type, and `task` is normalized task data. - NOTE: It's better to use `:on_task_change()` function. - -`fifo` queue does not support: - * task priorities (`pri`) - * task time to live (`ttl`) - * task time to execute (`ttr`) - * delayed execution (`delay`) - -Example: - -```lua - --- add a log record on task completion -local function otc_cb(task, stats_data) - if stats_data == 'delete' then - log.info("task %s is done", task[1]) - end -end - -queue.create_tube('tube_name', 'fifo', {temporary = true, on_task_change = otc_cb}) -queue.tube.tube_name:put('my_task_data_1') -queue.tube.tube_name:put('my_task_data_2') - -``` - -In the example above, the `otc_cb` function will be called 2 times, on each task -completion. Values for the callback arguments will be taken from the queue. - -## `fifottl` - a simple priority queue with support for task time to live - -The following options can be specified when creating a `fifottl` queue: - * `temporary` - boolean - if true, the contents of the queue do not persist -on disk - * `if_not_exists` - boolean - if true, no error will be returned if the tube -already exists - * `on_task_change` - function name - a callback to be executed on every -operation - -The following options can be specified when putting a task in a `fifottl` queue: - * `pri` - task priority (`0` is the highest priority and is the default) - * `ttl` - numeric - time to live for a task put into the queue, in -seconds. if `ttl` is not specified, it is set to infinity - (if a task exists in a queue for longer than ttl seconds, it is removed) - * `ttr` - numeric - time allotted to the worker to work on a task, in -seconds; if `ttr` is not specified, it is set to the same as `ttl` - (if a task is being worked on for more than `ttr` seconds, its status -is changed to 'ready' so another worker may take it) - * `delay` - time to wait before starting to execute the task, in seconds - -Example: - -```lua - -queue.create_tube('tube_name', 'fifottl', {temporary = true}) -queue.tube.tube_name:put('my_task_data', { ttl = 60.1, delay = 80 }) - -``` - -In the example above, the task has 60.1 seconds to live, but the start -of execution is delayed for 80 seconds. Thus the task actually will -exist for up to (60.1 + 80) 140.1 seconds. - -A smaller priority value indicates a higher priority, so a task with -priority 1 will be executed after a task with priority 0, if all other -options are equal. - -## `limfifottl` - a simple size-limited priority queue with support for task time to live - -Works same as fifottl, but has limitied size and put operation timeout. - -The following options can be specified when creating a `fifottl` queue: - * `temporary` - boolean - if true, the contents of the queue do not persist -on disk - * `if_not_exists` - boolean - if true, no error will be returned if the tube -already exists - * `on_task_change` - function name - a callback to be executed on every -operation - * `capacity` - number - limit size of the queue - -The following options can be specified when putting a task in a `fifottl` queue: - * `pri` - task priority (`0` is the highest priority and is the default) - * `ttl` - numeric - time to live for a task put into the queue, in -seconds. if `ttl` is not specified, it is set to infinity - (if a task exists in a queue for longer than ttl seconds, it is removed) - * `ttr` - numeric - time allotted to the worker to work on a task, in -seconds; if `ttr` is not specified, it is set to the same as `ttl` - (if a task is being worked on for more than `ttr` seconds, its status -is changed to 'ready' so another worker may take it) - * `delay` - time to wait before starting to execute the task, in seconds - * `timeout` - numeric - seconds to wait until queue has free space; if - `timeout` is not specified or time is up, and queue has no space, method return Nil - - -## `utube` - a queue with sub-queues inside - -The main idea of this queue backend is the same as in a `fifo` queue: -the tasks are executed in FIFO order. -However, tasks may be grouped into sub-queues. - -It is advised not to use `utube` methods inside transactions with -`read-confirmed` isolation level. It can lead to errors when trying to make -parallel tube methods calls with mvcc enabled. - -The following options can be specified when creating a `utube` queue: - * `temporary` - boolean - if true, the contents of the queue do not persist -on disk - * `if_not_exists` - boolean - if true, no error will be returned if the tube -already exists - * `on_task_change` - function name - a callback to be executed on every -operation - * `storage_mode` - string - one of - * `queue.driver.utube.STORAGE_MODE_DEFAULT` ("default") - default - implementation of `utube` - * `queue.driver.utube.STORAGE_MODE_READY_BUFFER` - ("ready_buffer") - allows processing `take` requests faster, but - by the cost of `put` operations speed. Right now this option is supported - only for `memtx` engine. - WARNING: this is an experimental storage mode. - - Here is a benchmark comparison of these two modes: - * Benchmark for simple `put` and `take` methods. 30k utubes are created - with a single task each. Task creation time is calculated. After that - 30k consumers are calling `take` + `ack`, each in the separate fiber. - Time to ack all tasks is calculated. The results are as follows: - - | | put (30k) | take+ack | - |---------|-----------|----------| - | default | 180ms | 1.6s | - | ready | 270ms | 1.7s | - * Benchmark for the busy utubes. 10 tubes are created. - Each contains 1000 tasks. After that, 10 consumers are created (each works - on his tube only, one tube — one consumer). Each consumer will - `take`, then `yield` and then `ack` every task from their utube - (1000 tasks each). - After that, we can also run this benchmark with 10k tasks on each utube, - 100k tasks and 150k tasks. But all that with 10 utubes and 10 consumers. - The results are as follows: - - | | 1k | 10k | 50k | 150k | - |---------|-------|------|------|-------| - | default | 53s | 1.5h | 100h | 1000h | - | ready | 450ms | 4.7s | 26s | 72s | - -The following options can be specified when putting a task in a `utube` -queue: - * `utube` - the name of the sub-queue. -Sub-queues split the task stream according to the sub-queue name: it is -not possible to take two tasks -out of a sub-queue concurrently, each sub-queue is executed in strict -FIFO order, one task at a time. - -`utube` queue does not support: - * task priorities (`pri`) - * task time to live (`ttl`) - * task time to execute (`ttr`) - * delayed execution (`delay`) - -Example: - -Imagine a web crawler, fetching pages from the Internet and finding URLs -to fetch more pages. -The web crawler is based on a queue, and each task in the queue refers -to a URL which the web crawler must download and process. -If the web crawler is split into many worker processes, then the same URL may -show up in the queue many times, because a single URL may be referred to by many -linking pages. -And the worker processes, working in parallel, can cause a denial-of-service on -the site of the URL. As a result, the web crawler can end up in the web -server's user-agent ban list -- not a desirable outcome. - -If the URL's domain name is used as a sub-queue name, this problem can be -solved: all the URLs with the same domain name can be fetched and processed -in strict FIFO order. - -## `utubettl` - extension of `utube` to support `ttl` - -This queue type is effectively a combination of `fifottl` and `utube`. - -It is advised not to use `utubettl` methods inside transactions with -`read-confirmed` isolation level. It can lead to errors when trying to make -parallel tube methods calls with mvcc enabled. - -The following options can be specified when creating a `utubettl` queue: - * `temporary` - boolean - if true, the contents of the queue do not persist -on disk - * `if_not_exists` - boolean - if true, no error will be returned if the tube -already exists - * `on_task_change` - function name - a callback to be executed on every -operation - * `storage_mode` - string - one of - * `queue.driver.utubettl.STORAGE_MODE_DEFAULT` ("default") - default - implementation of `utubettl` - * `queue.driver.utubettl.STORAGE_MODE_READY_BUFFER` - ("ready_buffer") - allows processing `take` requests faster, but - by the cost of `put` operations speed. Right now this option is supported - only for `memtx` engine. - WARNING: this is an experimental storage mode. - - Here is a benchmark comparison of these two modes: - * Benchmark for simple `put` and `take` methods. 30k utubes are created - with a single task each. Task creation time is calculated. After that - 30k consumers are calling `take` + `ack`, each in the separate fiber. - Time to ack all tasks is calculated. The results are as follows: - - | | put (30k) | take+ack | - |---------|-----------|----------| - | default | 200ms | 1.7s | - | ready | 320ms | 1.8s | - * Benchmark for the busy utubes. 10 tubes are created. - Each contains 1000 tasks. After that, 10 consumers are created (each works - on his tube only, one tube — one consumer). Each consumer will - `take`, then `yield` and then `ack` every task from their utube - (1000 tasks each). - After that, we can also run this benchmark with 10k tasks on each utube, - 100k tasks and 140k tasks. But all that with 10 utubes and 10 consumers. - The results are as follows: - - | | 1k | 10k | 50k | 140k | - |---------|-------|------|------|-------| - | default | 80s | 1.6h | 100h | 1000h | - | ready | 520ms | 5.4s | 28s | 83s | - -The following options can be specified when putting a task in a -`utubettl` queue: - * `pri` - task priority (`0` is the highest priority and is the default) - * `utube` - the name of the sub-queue - * `ttl` - numeric - time to live for a task put into the queue, in -seconds. if `ttl` is not specified, it is set to infinity - (if a task exists in a queue for longer than ttl seconds, it is removed) - * `ttr` - numeric - time allotted to the worker to work on a task, in -seconds; if `ttr` is not specified, it is set to the same as `ttl` - (if a task is being worked on for more than `ttr` seconds, its status -is changed to 'ready' so another worker may take it) - * `delay` - time to wait before starting to execute the task, in seconds - -# The underlying spaces - -The queue system consists of fibers, IPC channels, functions, and spaces. -Here is how queues map to spaces in a Tarantool database. - -The `_queue` space contains tuples for each queue and its properties. -This space is created automatically when the queue system is initialized for -the first time (for example, by "require 'queue'"), and is re-used on -later occasions. - -## Fields of the `_queue` space - -1. `tube` - the name of the queue -1. `tube_id` - queue ID, numeric -1. `space` - the name of a space associated with the queue, which -contains one tuple for each queue task -1. `type` - the queue type ('fifo', 'fifottl', 'utube', 'utubettl') -1. `opts` - additional options supplied when creating the queue, for -example 'ttl' - -The `_queue_consumers` temporary space contains tuples for each job -which is working on a queue. -Consumers may be simply waiting for tasks to be put in the queues. - -## Fields of the `_queue_consumers` space - -1. `connection_id` - connection ID of the client -1. `fid` - client fiber ID -1. `tube_id` - queue ID, referring to the `tube_id` field in the `_queue` -space; the client waits for tasks in this queue -1. `timeout` - the client wait timeout -1. `time` - the time when the client took a task - -The `_queue_taken_2` (`_queue_taken` is deprecated) space contains -tuples for each job which is processing a task in the queue. - -## Fields of the `_queue_taken_2` space - -1. `tube_id` - queue ID, to which the task belongs -1. `task_id` - task ID (of the task being taken) -1. `connection_id` - connection ID of the client, referring to the -`connection_id` field of the `_queue_consumers` space -1. `session_uuid` - session UUID (string) -1. `time` - the time when the client began to execute the task - -The `_queue_session_ids` space contains a map: connection id (box -session id) to the session UUID. This space is temporary if `in_replicaset` -is set to false. - -## Fields of the `_queue_session_ids` space - -1. `connection_id` - connection id (numeric) -2. `session_uuid` - session UUID (string) - -## Fields of the `_queue_shared_sessions` space - -1. `uuid` - session UUID (string) -2. `exp_time` - session expiration time (numeric) -3. `active` - session state (boolean) - -This space is temporary if `in_replicaset` is set to false. - -Also, there is a space which is associated with each queue, -which is named in the `space` field of the `_queue` space. -The associated space contains one tuple for each task. - -## Fields of the space associated with each queue - -1. task_id - numeric - see below -2. task_state - 'r' for ready, 't' for taken, etc. - see below -3. task_data - the contents of the task, usually a long string -x. (additional fields if the queue type has options for ttl, priority, -or delay) - -The `task_id` value is assigned to a task when it's inserted into a queue. -Currently, `task_id` values are simple integers for `fifo` and `fifottl` -queues. - -The `task_state` field takes one of the following values -(different queue types support different -sets of `task_state` values, so this is a superset): - -* 'r' - the task is **ready** for execution (the first consumer executing -a `take` request will get it) -* 't' - the task has been **taken** by a consumer -* '-' - the task has been **executed (done)** (a task is removed from the queue -after it has been executed, so this value will rarely be seen) -* '!' - the task is **buried** (disabled temporarily until further changes) -* '~' - the task is **delayed** for some time. - -For details on the state transitions, refer to [Task state diagram](#task-state-diagram). - -# Task state diagram - -The following diagram shows possible transitions between the [task states](#fields-of-the-space-associated-with-each-queue). -For information on the transition triggers, refer to: - -* [put()](#putting-a-task-in-a-queue) -* [release()](#releasing-a-task) -* [take()](#taking-a-task-from-the-queue-consuming) -* [kick()](#kicking-a-number-of-tasks) -* [bury()](#burying-a-task) -* [ack()](#acknowledging-the-completion-of-a-task) -* [delete()](#deleting-a-task) -* description of the `timeout`, `ttl timeout`, and `ttr timeout` options in -the sections of the corresponding [queue types](#queue-types). - -```mermaid -flowchart LR - INIT((" "))--> |"put()"| READY - INIT((" "))--> |"put('my_task_data', {delay = delay})"| DELAYED - READY--> |"take()"| TAKEN - READY--> |"delete() / ttl timeout"| DONE - READY--> |"bury()"| BURIED - TAKEN--> |"release() / ttr timeout"| READY - TAKEN--> |"release\n(id, {delay = delay})"| DELAYED - TAKEN--> |"ack() / delete()"| DONE - TAKEN--> |"bury()"| BURIED - BURIED--> |"delete() /\nttl timeout"| DONE - BURIED--> |"kick()"| READY - DELAYED--> |"timeout"| READY - DELAYED--> |"delete()"| DONE -``` - -# Queue state diagram - -Queue can be used in a master-replica scheme: - -There are five states for queue: -* INIT -* STARTUP -* RUNNING -* ENDING -* WAITING - -When the tarantool is launched for the first time, -the state of the queue is always `INIT` until `box.info.ro` is false. - -States switching scheme: -```mermaid -flowchart LR - I(("init"))-->S[startup] - S[startup]-->R[running] - W[waiting]--> |"(ro ->rw)"| S[startup] - R[running]--> |"(rw ->ro)"| E[ending] - E[ending]-->W[waiting] -``` - -Current queue state can be shown by using `queue.state()` method. - -In the `STARTUP` state, the queue is waiting for possible data synchronization -with other cluster members by the time of the largest upstream lag multiplied -by two. After that, all taken tasks are released, except for tasks with -session uuid matching shared sessions uuids. This makes possible to take -a task, switch roles on the cluster, and release the task within the timeout -specified by the `queue.cfg({ttr = N})` parameter. And the last step in the -`STARTUP` state is starting tube driver using new method called `start()`. - -In the `RUNNING` state, the queue is working as usually. The `ENDING` state calls -`stop()` method. in the `WAITING` state, the queue listens for a change in the -read_only flag. - -All states except `INIT` is controlled by new fiber called `queue_state_fiber`. - -# Installing - -There are three alternative ways of installation. -* Get the `tarantool_queue` package from a repository. For example, on -Ubuntu, say: sudo apt-get install tarantool-queue -* Take the Lua rock from rocks.tarantool.org. -* Take the source files from https://github.com/tarantool/queue, then -build and install. - -# Using the `queue` module - -## Initialization -```lua -queue = require 'queue' +# Queue API description +[![Build Status](https://travis-ci.org/tarantool/queue.svg?branch=stable)](https://travis-ci.org/tarantool/queue) + +A Tarantool/Box instance can serve as a Queue Manager, along +with any other database work necessary. + +A single properly configured Tarantool/Box space can store any +number of queues. Multiple spaces can be used as well - for +partitioning or logical separation of queues. + +Queues support task priority. Priority value lays in the range +[0, 255], with default value being 127. A higher value means higher +priority, lower value - lower priority. + +Each queue has one (currently) associated *fiber* taking care of +it. The fiber is started upon first access to the queue. The job +of the fiber is to monitor orphaned tasks, as well as prune and +clean the queue from obsolete tasks. + +To configure a space supporting queues, use the following parameters: + +```cfg +readahead = 16384 + +primary_port = 33020 +secondary_port = 33021 +admin_port = 33022 + + +space = [ + { + enabled = 1, + index = [ + { + type = "TREE", + unique = 1, + key_field = [ + { + fieldno = 0, + type = "STR" + } + ] + }, + { + type = "TREE", + unique = 0, + key_field = [ + { + fieldno = 1, # tube + type = "STR" + }, + { + fieldno = 2, # status + type = "STR" + }, + { + fieldno = 4, # ipri + type = "STR" + }, + { + fieldno = 5 # pri + type = "STR" + } + ] + }, + { + type = "TREE", + unique = 0, + key_field = [ + { + fieldno = 1, # tube + type = "STR" + }, + { + fieldno = 3, # next_event + type = "NUM64" + } + ] + } + ] + } +] ``` -The request "require 'queue'" causes automatic creation of -the `_queue` space, unless it already exists. The same request -also sets the necessary space triggers and other objects -associated with queues. -If the instance hasn't been configured yet (`box.cfg()` hasn't been called), -the initialization of the queue module will be deferred until the instance will -be configured ("lazy start"). For a good work of the queue, it's necessary to -run the instance in rw mode. If the instance run in ro mode, the initialization -of the queue will be deferred until the instance will be configured in rw mode. -After the instance has been started in rw mode and the queue has been -initialized, it's a bad idea to switch it to ro mode. In the case, an attempt to -do something with a persistent ("temporary" option set to false) queue will fail -(a temporary queue will work fine). In addition, in the case of mode has been -switched, triggers may fail (`_on_consumer_disconnect` for example), which may -cause an inconsistent state of the queue. As for the core drivers that use -background fibers (fifottl, limfifottl, utubettl) - they check the instance mode -on each iteration and will wait until the instance will be switched to rw mode. - -## Get the module version -```lua -queue._VERSION +If You want use method `queue.put_unique` you have to add additional +(fourth) index: + +```cfg + { + type = "TREE", + unique = 0, + key_field = [ + { + fieldno = 1, # tube + type = "STR" + }, + { + fieldno = 2, # status + type = "STR" + }, + { + fieldno = 12, # task data + type = "STR" + } + ] + } ``` -Returns the current version of the module. +It may also be desirable to tune server `readahead` configuration +variable if many consumers re-use the same socket for getting and +acknowledging tasks. -## Creating a new queue +The recommended value can be calculated as: -```lua -queue.create_tube(queue name, queue type [, {options} ]) ``` - -Creates a queue. - -The queue name must be alphanumeric and be up to 32 characters long. - -The queue type must be 'fifo', 'fifottl', 'utube', or 'utubettl'. - -The options, if specified, must be one or more of the options described above -(`temporary` and/or `ttl` and/or `ttr` and/or `pri`, depending on the queue -type). -The `ttr` and `ttl` options can be regarded as defaults, which may be overridden -when a task is put in a queue. - -Effect: a tuple is added in the `_queue` space, and a new associated space is -created. - -Example: `queue.create_tube('list_of_sites', 'fifo', {temporary = true})` - -## Set queue settings - -```lua -queue.cfg({options}) + consumers-per-socket * (256 + largest task size) ``` -Set queue settings. -If an invalid value or an unknown option is used, an error will be thrown. -Available `options`: -* `ttr` - time to release in seconds. The time after which, if there is no active -connection in the session, it will be released with all its tasks. -* `in_replicaset` - enable replication mode. Must be true if the queue is used -in master and replica mode. With replication mode enabled, the potential loss of -performance can be ~20% compared to single mode. Default value is false. +For example, if the largest task size is 256 bytes, and average +number of consumers per socket is 10, the recommended `readahead` +value must be at least 51 200 bytes. -## Session identify +## Terminology -```lua -queue.identify(session_uuid) -``` +* *Consumer* - a process, taking and executing tasks +* *Producer* - a process adding new tasks -In the queue the session has a unique UUID and many connections may share one -logical session. Also, the consumer can reconnect to the existing session during -the`ttr` time. -To get the UUID of the current session, call the `queue.identify()` -without parameters. -To connect to the existing session, call the `queue.identify(session_uuid)` -with the UUID of the session. -In case of attempt to use an invalid format UUID or expired UUID, an error will -be thrown. - -Be careful, UUID here is a 16-bit string generated by -[uuid.bin()](https://www.tarantool.io/en/doc/latest/reference/reference_lua/uuid/#lua-function.uuid.bin), -not an object of type UUID. - -Usage example: -Sometimes we need an ability to acknowledge a task after reconnect (because -retrying it is undesirable) or even acknowlegde using another connection. - -Example of code for connecting to the old session in case of reconnection: -```lua -local netbox = require('net.box') - -local conn = netbox.connect('localhost:1918', { reconnect_after = 5 }) -local session_uuid = conn:call('queue.identify') -conn:on_connect(function() - conn:call('queue.identify', {session_uuid}) -end) -``` +### Arguments of queue API functions -## Putting a task in a queue +* `space` (number) space id. To avoid confusion and broken statistics, + it's necessary to consistently use numbers to identify spaces, +* `tube` (string) - queue name, +* `delay` (number) - a delay between the moment a task is queued + and is executed, in seconds +* `ttl` (number) - task time to live, in seconds. If `delay` is + given along with `ttl`, the effective task time to live is + increased by the amount of `delay`, +* `ttr` (number) - task time to run, the maximal time allotted + to a consumer to execute a task, in seconds, +* `pri` (number) - task priority [0..255], +* `id` (string) - task id, +* `timeout` (number) - timeout in seconds for the Queue API function. -To insert a new task into a queue, use: +### Task states -```lua -queue.tube.tube_name:put(task_data [, {options} ]) -``` +* `ready` - a task is ready for execution, +* `delayed` - a task is awaiting task `delay` to expire, after + which it will become `ready`, +* `taken` - a task is taken by a consumer and is being executed, +* `done` - a task is complete (but not deleted, since the consumer + called `done` rather than `ack`), +* `buried` - a task is neither ready nor taken nor complete, it's + excluded (perhaps temporarily) from the list of tasks for + execution, but not deleted. -The `tube_name` must be the name which was specified by `queue.create_tube`. +### The format of task tuple -The `task_data` contents are the user-defined description of the task, -usually a long string. +Queue API functions, such as `put`, `take`, return a task. +The task consists of the following fields: -The options, if specified, must be one or more of the options described -above -(`ttl` and/or `ttr` and/or `pri` and/or `delay` and/or `utube`, depending on the queue -type). -If an option is not specified, the default is what was specified during -`queue.create_tube`, and if that was not specified, then the default is what -was described above for the queue type. Note: if the `delay` option is -specified, the delay time is added to the ttl time. +1. `id` (string) - task identifier +1. `tube` (string) - queue identifier +1. `status` (string) - task status +1. task data (all fields passed into `put`/`urgent` when + the task was created) -Effect: a new tuple is created in the queue's associated space, with -task_id = a number which is equal to the largest `task_id` so far, plus 1 -task_state = 'r' (ready) -task_data = whatever the user put in the `task_data` parameter +## API -Returns: the value of the new tuple in the queue's associated space, -also called the "created task". +### Producer -Example: queue.tube.list_of_sites:put('Your task is to do something', -{pri=2}) +#### queue.put(space, tube, delay, ttl, ttr, pri, ...) -After a task has been put in a queue, one of these things may happen: -it may be removed from the queue because its ttl (time to live) expires, -or it may be acted on by a worker (usually with a `take` request). +Enqueue a task. Returns a tuple, representing the new task. +The list of fields with task data ('...')is optional. -## Taking a task from the queue ("consuming") +#### queue.urgent(space, tube, delay, ttl, ttr, pri, ...) -```lua -queue.tube.tube_name:take([timeout]) -``` +Enqueue a task. The task will get the highest priority. +If `delay` is not zero, the function is equivalent to `put`. -Take a queue task. +### Consumer -The `take` request searches for a task in the queue or sub-queue -(that is, a tuple in the queue's associated space) -which has `task_state` = 'r' (ready), and `task_id` = a value lower -than any other tuple which also has `task_state` = 'r'. +#### queue.take(space, tube, timeout) -If there is no such task, and timeout was specified, then -the job waits until a task becomes ready or the timeout expires. +If there are tasks in the queue `ready` for execution, +take the highest-priority task. +Otherwise, wait for a `ready` task to appear in the queue, and, as +soon as it appears, mark it as `taken` and return to the consumer. +If there is a `timeout`, and the task doesn't appear until the +timeout expires, return 'nil' (a timeout of 0 returns immediately). +If timeout is not given or negative, wait indefinitely until a task +appears. -Effect: the value of `task_state` changes to 't' (taken). -The `take` request tells the system that the task is being worked on. -It should be followed by an `ack` request when the work is finished. -Additional effect: a tuple is added to the `_queue_taken_2` space. +All the time while the consumer is working on a task, it must keep +the connection to the server open. If a connection disappears while +the consumer is still working on a task, the task is put back on the +`ready` list. -Returns: the value of the taken tuple, or nil if none was found. -The value of the first field in the tuple (`task_id`) is important -for further requests. The value of the second field in the tuple -(`task_data`) is important as it presumably contains user-defined -instructions for what to do with the task. +#### queue.ack(space, id) -Example: t_value = queue.tube.list_of_sites:take(15) +Confirm completion of a task. Before marking a task as complete, +this function verifies that: -## Increasing TTR and/or TTL for tasks +* the task is `taken` and +* the consumer that is confirming the task is the one which took it. -```lua -queue.tube.tube_name:touch(task_id, increment) -``` +Consumer identity is established using a session identifier. In +other words, the task must be confirmed by the same connection +which took it. If verification fails, the function returns an +error. -Increase `ttr` of running task. Useful if you can't predict in advance -time needed to work on task. +On success, delete the task from the queue. -Effect: the value of `ttr` and `ttl` increased by `increment` seconds. If queue -does not support ttr, error will be thrown. If `increment` is lower than zero, -error will be thrown. If `increment` is zero or nil effect is noop. If current -`ttr` of task is 500 years or greater then operation is noop. +#### queue.release(space, id [, delay [, ttl ] ]) -Example: t_value = queue.tube.list_of_sites:touch(15, 60) +Return a task back to the queue: the task is not executed. +Additionally, a new time to live and re-execution delay can be +provided. +If `ttl` is not defined (or zero) the method won't prolong ttl. -## Acknowledging the completion of a task -```lua -queue.tube.tube_name:ack(task_id) -``` +#### queue.requeue(space, id) -The worker which has used 'take' to take the task should -use 'ack' to signal that the task has been completed. -The current `task_state` of the tuple should be 't' (taken), -and the worker issuing the `ack` request must have the same ID -as the worker which issued the `take` request. +Return a task to the queue, the task is not executed. Puts +the task at the end of the queue, so that it's executed only +after all existing tasks in the queue are executed. -Effect: the value of `task_state` changes to '-' (acknowledged). -Shortly after this, it may be removed from the queue automatically. +#### queue.bury(space, id) -If 'take' occurs but is not soon followed by 'ack' --- that is, if ttr (time to run) expires, or if the worker disconnects -- -the effect is: `task_state` is changed from -'t' (taken) back to 'r' (ready). -This effect is the same as what would happen with a `release` request. +Mark a task as `buried`. This special status excludes +the task from the active list, until it's `dug up`. +This function is useful when several attempts to execute a task +lead to a failure. Buried tasks can be monitored by the queue +owner, and treated specially. -Example: queue.tube.list_of_sites:ack(15) +#### queue.done(space, id, ...) -## Releasing a task +Mark a task as complete (`done`), but don't delete it. +Replaces task data with the supplied fields ('...'). -```lua -queue.tube.tube_name:release(task_id, opts) -``` +### Common functions (neither producer nor consumer). -Put the task back in the queue. -A worker which has used 'take' to take a task, -but cannot complete it, may make a `release` request -instead of an `ack` request. Effectively, 'ack' implies -successful completion of a taken task, and 'release' implies -unsuccessful completion of a taken task. +#### queue.dig(space, id) -Effect: the value of `task_state` changes to 'r' (ready). -After this, another worker may take it. -This is an example of a situation where, due to user intervention, -a task may not be successfully completed in strict FIFO order. +'Dig up' a buried task, after checking that the task +is buried. The task status is changed to `ready`. -Example: queue.tube.list_of_sites:release(15, {delay=10}) +#### queue.kick(space, tube [, count] ) -Note: in the above example, the delay option means -"the task cannot be executed again for 10 seconds". +'Dig up' `count` tasks in a queue. If `count` is not given, +digs up just one buried task. -## Peeking at a task +#### queue.unbury(space, id) -```lua -queue.tube.tube_name:peek(task_id) -``` +An alias to `dig`. -Look at a task without changing its state. +#### queue.delete(space, id) -Effect: this is the same as getting a tuple from the space -associated with the queue: box.space.tube_name:select(task_id). +Delete a task from the queue (regardless of task state or status). -Returns: the tuple of the task. +#### queue.truncate(space, tube) -Example: queue.tube.list_of_sites:peek(15) +Truncate a tube. Return the number of deleted tasks. -## Burying a task +#### queue.meta(space, id) -```lua -queue.tube.tube_name:bury(task_id) -``` +Return taks metadata: -If it becomes clear that a task cannot be executed -in the current circumstances, you can "bury" the task --- that is, disable it until the circumstances change. +1. `id` (string) - task id +1. `tube` (string) - queue id +1. `status` (string) - task status +1. `event` (time64) - time of the next important event in task + life time, for example, when `ttl` or `ttr` expires, in seconds + since start of the UNIX epoch +1. `ipri` (string) - internal value of the task priority +1. `pri` (string) - task priority as set when the task was added + to the queue +1. `cid` (number) - consumer id, of the consumer which took the + task (only if the task is `taken`) +1. `created` (time64) - time when the task was created (seconds + since start of the UNIX epoch). +1. `ttl` (time64) - task time to live +1. `ttr` (time64) - task time to run +1. `cbury` (count) - how many times the task was buried +1. `ctaken` (Ч) - how many times the task was taken +1. `now` (time64) - time recorded when the meta was called -Effect: the value of `task_state` changes to '!' (buried). -Since '!' is not equal to 'r' (ready), the task can no longer be taken. -Since '!' is not equal to '-' (complete), the task will not be deleted. -The only thing that can affect a buried task is a `kick` request. +#### queue.peek(space, id) -Returns: the tuple value. +Return a task by task id. Returned tuple has the following +fields: -Example: queue.tube.list_of_sites:bury(15) +1. `id` (string) - task identifier +1. `tube` (string) - queue identifier +1. `status` (string) - task status +1. task data (all fields passed into `put`/`urgent` when + the task was created). -## Kicking a number of tasks - -```lua -queue.tube.tube_name:kick(count) -``` +#### queue.statistics() -Reverse the effect of a `bury` request on one or more tasks. +Return queue module statistics accumulated since server start. +The statistics is broken down by queue id. Only queues on which +there was some activity are included in the output. -Effect: the value of `task_state` changes from '!' (buried) -to 'r' (ready), for one or more tasks. - -Returns: number of tasks actually kicked. - -Example: queue.tube.list_of_sites:kick(99) -(this will change up to 99 buried tasks) - -## Deleting a task - -```lua -queue.tube.tube_name:delete(task_id) -``` - -Delete the task identified by `task_id`. - -Effect: the current state of `task_state` is not checked. -The task is removed from the queue. - -Example: queue.tube.list_of_sites:delete(15) - -## Dropping a queue - -```lua -queue.tube.tube_name:drop() -``` - -Reverse the effect of a `create` request. - -Effect: remove the tuple from the `_queue` space, -and drop the space associated with the queue. - -## Releasing all taken tasks - -```lua -queue.tube.tube_name:release_all() -``` - -Forcibly returns all taken tasks to a ready state. - -## Getting statistics - -```lua -queue.statistics( [queue name] ) -``` - -Show the number of tasks in a queue broken down by `task_state`, and the number -of requests broken down by the type of request. If the queue name is not -specified, show these numbers for all queues. -Statistics are temporary, they are reset whenever the Tarantool server restarts. - -Example: - -```lua -queue.tube.tube_name:on_task_change(callback) -``` - -Replace old `on_task_change` callback or set the new one. Previously set -callback is returned. - -Get statistics for given tube: - -```lua -queue.statistics('list_of_sites') ---- -- tasks: - taken: 0 - buried: 0 - ready: 0 - done: 2 - delayed: 0 - total: 0 - calls: - ack: 1 - take: 1 - kick: 1 - bury: 1 - put: 2 - delete: 1 -... -``` - -## Queue and replication - -Usage example: - -```lua --- Instance file for the master. -queue = require("queue") --- Queue is in replicaset. --- Clean up session after 5 minutes after disconnect. -queue.cfg({ttr = 300, in_replicaset = true}) - -box.cfg{ - listen = 3301, - replication = {'replicator:password@127.0.0.1:3301', -- Master URI. - 'replicator:password@127.0.0.1:3302'}, -- Replica URI. - read_only = false, -} - -box.once("schema", function() - box.schema.user.create('replicator', {password = 'password'}) - box.schema.user.grant('replicator', 'replication') -- grant replication role -end) - -require('console').start() -os.exit() -``` - -```lua --- Instance file for the replica. -queue = require("queue") --- Queue is in replicaset. --- Clean up session after 5 minutes after disconnect. -queue.cfg({ttr = 300, in_replicaset = true}) -box.cfg{ - listen = 3302, - replication = {'replicator:password@127.0.0.1:3301', -- Master URI. - 'replicator:password@127.0.0.1:3302'}, -- Replica URI. - read_only = true -} - -require('console').start() -os.exit() -``` - -Start master and replica instances and check queue state: - -Master: -```sh -tarantool> queue.state() ---- -- RUNNING -... -``` - -Replica: -```sh -tarantool> queue.state() ---- -- INIT -... -``` - -Now reverse the `read_only` setting of the master and replica and check the -status of the queue again. - -Master: -```sh -tarantool> box.cfg({read_only = true}) -tarantool> queue.state() ---- -- WAITING -... -``` - -Replica: -```sh -tarantool> box.cfg({read_only = false}) -tarantool> queue.state() ---- -- RUNNING -... -``` +The format of the statistics is a sequence of rows, where each +odd row is the name of a statistical parameter, and the +next even row is the value. -# Implementation details - -The implementation is based on the common functions for all queues: - -1. controlling the `consumers` (watching connection state/wakeup) -1. similarities of the API -1. spaces to support each tube -1. etc - -Each new queue has a "driver" to support it. - -## Queue drivers - -Mandatory requirements - -1. The driver works with tuples. The only thing the driver needs -to know about the tuples is their first two fields: `id` and `state`. -1. Whenever the driver notices that a task state has changed, it must -notify the framework about the change. -1. The driver must not throw exceptions, unless the driver API is misused. -I.e. for normal operation, even errors during normal operation, there -should be no exceptions. - -Registering a custom driver - -`register_driver(driver_name, tube_ctr)` - queue method is used to register - a custom driver. The arguments are: - * driver_name: unique driver name. Must be different from the core drivers - names. - * tube_ctr: implementation of tube control methods("create_space" and "new"). - -## Driver API - -Driver class must implement the following API: - -1. `new` (constructs an instance of a driver), takes: - * the space object, in which the driver must store its tasks - * a callback to notify the main queue framework on a task state change - (`on_task_change`) - * options of the queue (a Lua table) -1. `create_space` - creates the supporting space. The arguments are: - * space name - * space options -1. `start` - initialize internal resources if any, e.g. start fibers. -1. `stop` - clean up internal resources if any, e.g. stop fibers. - -To sum up, when the user creates a new queue, the queue framework -passes the request to the driver, asking it to create a space to -support this queue, and then creates a driver instance, passing to it -the created space object. - -The same call sequence is used when the queue is "restarted" after -Tarantool server restart. - -The driver instance returned by the `new` method must provide the following -API: - -* `tube:normalize_task(task)` - converts the task tuple to the object -which is passed on to the user (removes the administrative fields) -* `tube:put(data[, opts])` - puts a task into the queue. -Returns a normalized task which represents a tuple in the space -* `tube:take()` - sets the task state to 'in progress' and returns the task. -If there are no 'ready' tasks in the queue, returns nil. -* `tube:delete(task_id)` - deletes a task from the queue. -Returns the original task with a state changed to 'done' -* `tube:release(task_id, opts)` - puts a task back to the queue (in the 'ready' - state) -* `tube:bury(task_id)` - buries a task -* `tube:kick(count)` - digs out `count` tasks -* `tube:peek(task_id)` - return the task state by ID -* `tube:touch(task_id, delta)` - increases `ttr` and `ttl` of the task by delta -seconds. If queue does not support `ttr`, error will be thrown. Returns the task -* `tube:tasks_by_state(task_state)` - return the iterator to tasks in a certain state -* `tube:truncate()` - delete all tasks from the tube. Note that `tube:truncate` -must be called only by the user who created this tube (has space ownership) OR -under a `setuid` function. Read more about `setuid` functions -[here](http://tarantool.org/doc/book/box/authentication.html?highlight=function#functions-and-the-func-space). - - -[testing-actions-badge]: https://github.com/tarantool/queue/actions/workflows/fast_testing.yml/badge.svg -[testing-actions-url]: https://github.com/tarantool/queue/actions/workflows/fast_testing.yml - -[packaging-actions-badge]: https://github.com/tarantool/queue/actions/workflows/packaging.yml/badge.svg -[packaging-actions-url]: https://github.com/tarantool/queue/actions/workflows/packaging.yml - -[publish-actions-badge]: https://github.com/tarantool/queue/actions/workflows/publish.yml/badge.svg -[publish-actions-url]: https://github.com/tarantool/queue/actions/workflows/publish.yml diff --git a/benchmark/channel.pl b/benchmark/channel.pl new file mode 100644 index 00000000..68457d1c --- /dev/null +++ b/benchmark/channel.pl @@ -0,0 +1,89 @@ +#!/usr/bin/perl + +use warnings; +use strict; +use utf8; +use open qw(:std :utf8); +use lib qw(lib ../lib); + +use Encode qw(decode encode); +use Cwd 'cwd'; +use File::Spec::Functions 'catfile'; +use feature 'state'; + +use Coro; +use DR::Tarantool ':all'; +use DR::Tarantool::StartTest; +use Time::HiRes 'time'; +use Data::Dumper; + +my $t = DR::Tarantool::StartTest->run( + cfg => catfile(cwd, 'tarantool.cfg'), + script_dir => catfile(cwd, 'benchmark') +); + +sub tnt { + our $tnt; + unless(defined $tnt) { + $tnt = coro_tarantool + host => 'localhost', + port => $t->primary_port, + spaces => {} + ; + } + return $tnt; +}; + +tnt->ping; + +my (@f); +my $cnt = 2500; +my $done = 0; + +my $started = time; + +{ + my @tp = 1 .. $cnt; + my @tg = 1 .. $cnt; + while (@tp or @tg) { + for (0 .. int rand 100) { + next unless @tp; + my $delay = rand; + $delay = 0 if 20 > rand 100; + push @f => async { + eval { + tnt->call_lua('put', [ $delay, shift @tp ]); + 1; + } or warn $@; + }; + cede; + } + + for (0 .. int rand 100) { + next unless @tg; + my $delay = rand; + $delay = 0 if 20 > rand 100; + push @f => async { + eval { + tnt->call_lua('get', [ $delay, shift @tg ]); + 1; + } or warn $@; + }; + cede; + } + } + + $_->join for @f; + @f = (); + + my $interval = time - $started; + $done += $cnt; + + printf "Done %d pair of tasks in %3.2f second (%3.4f s/p and %d p/s)\n", + $done, + $interval, + $interval / $done, + $done / $interval + ; + redo; +} diff --git a/benchmark/init.lua b/benchmark/init.lua new file mode 100644 index 00000000..5514d5c3 --- /dev/null +++ b/benchmark/init.lua @@ -0,0 +1,19 @@ +ch = box.ipc.channel(10) + +function put(delay, data) + if tonumber(delay) > 0 then + box.fiber.sleep(tonumber(delay)) + end + return ch:put(data) +end + +function get(delay) + if tonumber(delay) > 0 then + box.fiber.sleep(tonumber(delay)) + end + return ch:get() +end + +function ping(...) + return ... +end diff --git a/benchmark/ping.pl b/benchmark/ping.pl new file mode 100644 index 00000000..bfa47975 --- /dev/null +++ b/benchmark/ping.pl @@ -0,0 +1,59 @@ +#!/usr/bin/perl + +use warnings; +use strict; +use utf8; +use open qw(:std :utf8); +use lib qw(lib ../lib); + +use Encode qw(decode encode); +use Cwd 'cwd'; +use File::Spec::Functions 'catfile'; +use feature 'state'; + +# use Coro; +use DR::Tarantool ':all'; +use DR::Tarantool::StartTest; +use Time::HiRes 'time'; +use Data::Dumper; + +my $t = DR::Tarantool::StartTest->run( + cfg => catfile(cwd, 'tarantool.cfg'), + script_dir => catfile(cwd, 'benchmark') +); + +sub tnt { + our $tnt; + unless(defined $tnt) { + $tnt = coro_tarantool + host => 'localhost', + port => $t->primary_port, + spaces => {} + ; + } + return $tnt; +}; + +$| = 1; + +my $process = 1; +$SIG{INT} = $SIG{TERM} = sub { print "\ncaught SIGexit\n"; $process = 0 }; + +my $done = 0; +my $total_time = 0; +while($process) { + my $started = time; + for (1 .. 10000) { + die "Can't ping tarantool" unless tnt->call_lua('ping', [1,2,3]); + } + + $done += 10000; + my $done_time = time - $started; + $total_time += $done_time; + printf "Done %d pings in %3.2f second (%f s/p and %d p/s)\n", + $done, + $done_time, + $total_time / $done, + $done / $total_time + ; +} diff --git a/benchmark/pqueue-mix.pl b/benchmark/pqueue-mix.pl new file mode 100644 index 00000000..14e4c7fb --- /dev/null +++ b/benchmark/pqueue-mix.pl @@ -0,0 +1,81 @@ +#!/usr/bin/perl + +use warnings; +use strict; +use utf8; +use open qw(:std :utf8); +use lib qw(lib ../lib); + +use Encode qw(decode encode); +use Cwd 'cwd'; +use File::Spec::Functions 'catfile'; +use feature 'state'; + +use Coro; +use DR::Tarantool ':all'; +use DR::Tarantool::StartTest; +use Time::HiRes 'time'; +use Data::Dumper; +use Coro::AnyEvent; +use DR::TarantoolQueue; + +my $t = DR::Tarantool::StartTest->run( + cfg => catfile(cwd, 'tarantool.cfg'), + script_dir => catfile(cwd) +); + +my $q = DR::TarantoolQueue->new( + host => '127.0.0.1', + port => $t->primary_port, + space => 0, + tube => 'test_tube' +); + +use constant ITERATIONS => 1000; + +my ($done, $total_time) = (0) x 4; +my $process = 1; + +$SIG{INT} = $SIG{TERM} = sub { + print "\nSIGING received\n"; + $t->kill unless $process; + $process = 0; +}; + +while($process) { + + my (@f, %t); + my $started = time; + for (1 .. ITERATIONS) { + push @f => async { + my $task = $q->put(data => { num => rand }); + $t{ $task->id }++; + }; + push @f => async { + my $task = $q->take; + $task->ack; + $t{ $task->id }++; + } + } + + $_->join for @f; @f = (); + + $total_time += time - $started; + $done += ITERATIONS; + + if (scalar keys %t != ITERATIONS) { + print "Wrong results count\n"; + last; + } + if (ITERATIONS != grep { $_ == 2 } values %t) { + print "Not all tasks were processed twice\n"; + last; + } + + printf "Done %d sessions in %3.2f seconds (%d r/s, %f s/r)\n", + $done, + $total_time, + $done / $total_time, + $total_time / $done + ; +} diff --git a/benchmark/pqueue.pl b/benchmark/pqueue.pl new file mode 100644 index 00000000..04f69420 --- /dev/null +++ b/benchmark/pqueue.pl @@ -0,0 +1,97 @@ +#!/usr/bin/perl + +use warnings; +use strict; +use utf8; +use open qw(:std :utf8); +use lib qw(lib ../lib); + +use Encode qw(decode encode); +use Cwd 'cwd'; +use File::Spec::Functions 'catfile'; +use feature 'state'; + +use Coro; +use DR::Tarantool ':all'; +use DR::Tarantool::StartTest; +use Time::HiRes 'time'; +use Data::Dumper; +use Coro::AnyEvent; +use DR::TarantoolQueue; + +my $t = DR::Tarantool::StartTest->run( + cfg => catfile(cwd, 'tarantool.cfg'), + script_dir => catfile(cwd) +); + +my $q = DR::TarantoolQueue->new( + host => '127.0.0.1', + port => $t->primary_port, + space => 0, + tube => 'test_tube' +); + +use constant ITERATIONS => 1000; + +my ($done, $total_put, $total_take_ack, $total_time) = (0) x 4; +my $process = 1; + +$SIG{INT} = $SIG{TERM} = sub { + print "\nSIGING received\n"; + $t->kill unless $process; + $process = 0; +}; + +while($process) { + + my (@f, %t); + my $started = time; + for (1 .. ITERATIONS) { + push @f => async { + my $task = $q->put(data => { num => rand }); + $t{ $task->id }++; + }; + } + + $_->join for @f; @f = (); + my $put_time = time - $started; + + $started = time; + for (1 .. ITERATIONS) { + push @f => async { + my $task = $q->take; + $task->ack; + $t{ $task->id }++; + } + } + $_->join for @f; @f = (); + my $take_ack_time = time - $started; + + $total_put += $put_time; + $total_take_ack += $take_ack_time; + $total_time += $put_time + $take_ack_time; + $done += ITERATIONS; + + if (scalar keys %t != ITERATIONS) { + print "Wrong results count\n"; + last; + } + if (ITERATIONS != grep { $_ == 2 } values %t) { + print "Not all tasks were processed twice\n"; + last; + } + + printf "\nDone %d sessions in %3.2f seconds (%d r/s, %f s/r)\n", + $done, + $total_time, + $done / $total_time, + $total_time / $done + ; + + printf " put: %6d r/s, %1.6f s/r, take/ack: %6d r/s, %1.6f s/r\n", + $done / $total_put, + $total_put / $done, + $done / $total_take_ack, + $total_take_ack / $done, + ; +} diff --git a/benchmark/queue.pl b/benchmark/queue.pl new file mode 100644 index 00000000..f933dad2 --- /dev/null +++ b/benchmark/queue.pl @@ -0,0 +1,122 @@ +#!/usr/bin/perl + +use warnings; +use strict; +use utf8; +use open qw(:std :utf8); +use lib qw(lib ../lib); + +use Encode qw(decode encode); +use Cwd 'cwd'; +use File::Spec::Functions 'catfile'; +use feature 'state'; + +use Coro; +use DR::Tarantool ':all'; +use DR::Tarantool::StartTest; +use Time::HiRes 'time'; +use Data::Dumper; +use Coro::AnyEvent; + +my $t = DR::Tarantool::StartTest->run( + cfg => catfile(cwd, 'tarantool.cfg'), + script_dir => catfile(cwd) +); + +sub tnt { + our $tnt; + unless(defined $tnt) { + $tnt = coro_tarantool + host => 'localhost', + port => $t->primary_port, + spaces => {} + ; + } + return $tnt; +}; + +tnt->ping; + +my $done = 0; +my $total_time = 0; +my $total_put = 0; +my $total_take_ack = 0; +my $process = 1; + +$SIG{INT} = $SIG{TERM} = sub { + print "\nSIGING received\n"; + $t->kill unless $process; + $process = 0; +}; + + +use constant ITERATIONS => 1000; + + +while($process) { + + my $put_time = 0; + my $take_ack_time = 0; + my $start_time = time; + my (@f, %t); + for (my $i = 0; $i < ITERATIONS; $i++) { + push @f => async { + my $tuple = tnt->call_lua('queue.put', + [ 0, 'tube', 0, 10, 5, 1, 'task body' ]); + $t{ $tuple->raw(0) }++; + }; + } + + $_->join for @f; + @f = (); + $put_time = time - $start_time; + $start_time = time; + + for (my $i = 0; $i < ITERATIONS; $i++) { + push @f => async { + my $tuple = tnt->call_lua('queue.take', [ 0, 'tube', 3 ]); + $t{ $tuple->raw(0) }++ if $tuple; + + tnt->call_lua('queue.ack', [ 0, $tuple->raw(0) ]); + }; + + } + + $_->join for @f; + @f = (); + + $take_ack_time = time - $start_time; + + my $done_time = $take_ack_time + $put_time; + $total_time += $done_time; + $done += ITERATIONS; + + $total_put += $put_time; + $total_take_ack += $take_ack_time; + + if (scalar keys %t != ITERATIONS) { + print "Wrong results count\n"; + last; + } + if (ITERATIONS != grep { $_ == 2 } values %t) { + print "Not all tasks were processed twice\n"; + last; + } + + printf "\nDone %d sessions in %3.2f seconds (%d r/s, %f s/r)\n", + $done, + $total_time, + $done / $total_time, + $total_time / $done + ; + + printf " put: %6d r/s, %1.6f s/r, take/ack: %6d r/s, %1.6f s/r\n", + $done / $total_put, + $total_put / $done, + $done / $total_take_ack, + $total_take_ack / $done, + ; + +} + +warn $t->log if $ENV{DEBUG}; diff --git a/cmake/FindTarantool.cmake b/cmake/FindTarantool.cmake deleted file mode 100644 index c9a9ba3c..00000000 --- a/cmake/FindTarantool.cmake +++ /dev/null @@ -1,52 +0,0 @@ -# Define GNU standard installation directories -include(GNUInstallDirs) - -macro(extract_definition name output input) - string(REGEX MATCH "#define[\t ]+${name}[\t ]+\"([^\"]*)\"" - _t "${input}") - string(REGEX REPLACE "#define[\t ]+${name}[\t ]+\"(.*)\"" "\\1" - ${output} "${_t}") -endmacro() - -find_path(TARANTOOL_INCLUDE_DIR tarantool/module.h - HINTS ENV TARANTOOL_DIR -) - -if(TARANTOOL_INCLUDE_DIR) - set(_config "-") - file(READ "${TARANTOOL_INCLUDE_DIR}/tarantool/module.h" _config0) - string(REPLACE "\\" "\\\\" _config ${_config0}) - unset(_config0) - extract_definition(PACKAGE_VERSION TARANTOOL_VERSION ${_config}) - extract_definition(INSTALL_PREFIX _install_prefix ${_config}) - unset(_config) -endif() - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(TARANTOOL - REQUIRED_VARS TARANTOOL_INCLUDE_DIR VERSION_VAR TARANTOOL_VERSION) -if(TARANTOOL_FOUND) - set(TARANTOOL_INSTALL_LIBDIR "${CMAKE_INSTALL_LIBDIR}/tarantool") - set(TARANTOOL_INSTALL_LUADIR "${CMAKE_INSTALL_DATADIR}/tarantool") - set(TARANTOOL_INCLUDE_DIRS "${TARANTOOL_INCLUDE_DIR}" - "${TARANTOOL_INCLUDE_DIR}/tarantool/") - - if (NOT "${CMAKE_INSTALL_PREFIX}" STREQUAL "/usr/local" AND - NOT "${CMAKE_INSTALL_PREFIX}" STREQUAL "${_install_prefix}") - message(WARNING "Provided CMAKE_INSTALL_PREFIX is different from " - "CMAKE_INSTALL_PREFIX of Tarantool. You might need to set " - "corrent package.path/package.cpath to load this module or " - "change your build prefix:" - "\n" - "cmake . -DCMAKE_INSTALL_PREFIX=${_install_prefix}" - "\n" - ) - endif () - if (NOT TARANTOOL_FIND_QUIETLY AND NOT FIND_TARANTOOL_DETAILS) - set(FIND_TARANTOOL_DETAILS ON CACHE INTERNAL "Details about TARANTOOL") - message(STATUS "Tarantool LUADIR is ${TARANTOOL_INSTALL_LUADIR}") - message(STATUS "Tarantool LIBDIR is ${TARANTOOL_INSTALL_LIBDIR}") - endif () -endif() -mark_as_advanced(TARANTOOL_INCLUDE_DIRS TARANTOOL_INSTALL_LIBDIR - TARANTOOL_INSTALL_LUADIR) diff --git a/debian/.gitignore b/debian/.gitignore deleted file mode 100644 index 53b5bf54..00000000 --- a/debian/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -tarantool-queue/ -files -stamp-* -*.substvars -*.log diff --git a/debian/changelog b/debian/changelog index 4ae7fd81..6471c910 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,149 @@ -tarantool-queue (1.0.0) unstable; urgency=medium +dr-tarantool-queue (0.27) nowtaxi; urgency=low - * Initial release + * restart_check works only if restart defined, too. - -- Roman Tsisyk Thu, 18 Feb 2016 19:58:35 +0300 + -- Dmitry E. Oboukhov Tue, 03 Mar 2015 15:41:48 +0300 + +dr-tarantool-queue (0.26) nowtaxi; urgency=low + + * Fix tests (negative timeout check), add restart_check trigger. + + -- Dmitry E. Oboukhov Tue, 03 Mar 2015 15:33:46 +0300 + +dr-tarantool-queue (0.25-1) nowtaxi; urgency=medium + + * Sync client uses realsync connection if it is available. + + -- Dmitry E. Oboukhov Thu, 17 Apr 2014 14:18:14 +0400 + +dr-tarantool-queue (0.24-1) nowtaxi; urgency=medium + + * Add method queue.put_unique. + + -- Dmitry E. Oboukhov Thu, 06 Feb 2014 14:25:13 +0400 + +dr-tarantool-queue (0.23-1) nowtaxi; urgency=low + + * Save error message before sendmail. + + -- Dmitry E. Oboukhov Tue, 12 Nov 2013 23:12:29 +0400 + +dr-tarantool-queue (0.22-1) nowtaxi; urgency=low + + * Drop 'disposition' for the first mailpart. + + -- Dmitry E. Oboukhov Tue, 12 Nov 2013 23:08:08 +0400 + +dr-tarantool-queue (0.21-1) nowtaxi; urgency=low + + * Add debug information in mail. + + -- Dmitry E. Oboukhov Tue, 12 Nov 2013 23:00:06 +0400 + +dr-tarantool-queue (0.20-1) nowtaxi; urgency=low + + * Typo in MIME::Lite attributes. + + -- Dmitry E. Oboukhov Tue, 12 Nov 2013 22:53:06 +0400 + +dr-tarantool-queue (0.19-1) nowtaxi; urgency=low + + * Encode subject as utf8 before sendmail. + + -- Dmitry E. Oboukhov Tue, 12 Nov 2013 22:47:55 +0400 + +dr-tarantool-queue (0.18-1) nowtaxi; urgency=low + + * If worker crashes it can send email. + + -- Dmitry E. Oboukhov Tue, 12 Nov 2013 22:36:24 +0400 + +dr-tarantool-queue (0.17-1) nowtaxi; urgency=low + + * User can define tarantool connection outside queue. + + -- Dmitry E. Oboukhov Sat, 26 Oct 2013 13:11:19 +0400 + +dr-tarantool-queue (0.16-1) nowtaxi; urgency=low + + * Space is always converted to luanumber. + + -- Dmitry E. Oboukhov Wed, 15 May 2013 15:43:05 +0400 + +dr-tarantool-queue (0.15-1) nowtaxi; urgency=low + + * Fix bug in process_tube (it can write wrong statistics) + + -- Dmitry E. Oboukhov Fri, 19 Apr 2013 02:25:17 +0400 + +dr-tarantool-queue (0.14-2) nowtaxi; urgency=low + + * Update MANIFEST (add forgotten files). + + -- Dmitry E. Oboukhov Sun, 14 Apr 2013 21:30:19 +0400 + +dr-tarantool-queue (0.12-1) nowtaxi; urgency=low + + * Add some methods to DR::TarantoolQueue::Worker. + + -- Dmitry E. Oboukhov Sun, 14 Apr 2013 18:05:25 +0400 + +dr-tarantool-queue (0.11-1) nowtaxi; urgency=low + + * Add DR::TarantoolQueue::Worker module. + + -- Dmitry E. Oboukhov Sun, 14 Apr 2013 14:47:59 +0400 + +dr-tarantool-queue (0.10-1) nowtaxi; urgency=low + + * Fix queue.meta (now is returned as NUM64 as described in README). + + -- Dmitry E. Oboukhov Mon, 01 Apr 2013 14:24:55 +0400 + +dr-tarantool-queue (0.09-1) nowtaxi; urgency=low + + * Fix always use Coro + + -- Roman V. Nikolaev Sat, 26 Jan 2013 18:37:31 +0400 + +dr-tarantool-queue (0.08-1) nowtaxi; urgency=low + + * Multiply queue config in driver. + + -- Dmitry E. Oboukhov Tue, 22 Jan 2013 23:39:57 +0400 + +dr-tarantool-queue (0.07-1) nowtaxi; urgency=low + + * Task can be created with wrong JSON. + + -- Dmitry E. Oboukhov Mon, 21 Jan 2013 17:30:52 +0400 + +dr-tarantool-queue (0.06-1) nowtaxi; urgency=low + + * Reconnect options for connector. + + -- Dmitry E. Oboukhov Mon, 21 Jan 2013 15:10:37 +0400 + +dr-tarantool-queue (0.05-1) nowtaxi; urgency=low + + * Fix take without timeout. + + -- Dmitry E. Oboukhov Mon, 21 Jan 2013 01:47:13 +0400 + +dr-tarantool-queue (0.04-1) nowtaxi; urgency=low + + * queue.relese don't prolong TTL by default. + + -- Dmitry E. Oboukhov Sun, 20 Jan 2013 23:52:02 +0400 + +dr-tarantool-queue (0.03-1) nowtaxi; urgency=low + + * Add some absent 'use' directives. + + -- Dmitry E. Oboukhov Sun, 20 Jan 2013 23:35:32 +0400 + +dr-tarantool-queue (0.02-1) nowtaxi; urgency=low + + * Initial Release. + + -- Dmitry E. Oboukhov Fri, 18 Jan 2013 01:52:32 +0400 diff --git a/debian/compat b/debian/compat index ec635144..45a4fb75 100644 --- a/debian/compat +++ b/debian/compat @@ -1 +1 @@ -9 +8 diff --git a/debian/control b/debian/control index 9688dee1..69665d11 100644 --- a/debian/control +++ b/debian/control @@ -1,18 +1,40 @@ -Source: tarantool-queue -Priority: optional +Source: dr-tarantool-queue Section: database -Maintainer: Roman Tsisyk -Build-Depends: debhelper (>= 9), - tarantool-dev (>= 1.7), -# For /usr/bin/prove - perl (>= 5.10.0) -Standards-Version: 3.9.6 -Homepage: https://github.com/tarantool/queue -Vcs-Git: git://github.com/tarantool/queue.git -Vcs-Browser: https://github.com/tarantool/queue +Priority: optional +Maintainer: Dmitry E. Oboukhov +Uploaders: Roman V. Nikolaev +Build-Depends: debhelper (>= 8) +Build-Depends-Indep: libcoro-perl, + libdr-tarantool-perl, + libjson-xs-perl, + libmouse-perl, + libcoro-perl, + perl, + tarantool (>= 1.4.8+20130110) +Standards-Version: 3.9.3 -Package: tarantool-queue +Package: libdr-tarantoolqueue-perl +Section: perl Architecture: all -Depends: tarantool (>= 1.7), ${misc:Depends} -Description: Persistent in-memory queues for Tarantool - A collection of persistent queue implementations for Tarantool. +Suggests: dr-tarantool-queue +Depends: ${misc:Depends}, ${perl:Depends}, + libcoro-perl, + libdr-tarantool-perl, + libjson-xs-perl, + libmouse-perl, + libcoro-perl +Homepage: http://search.cpan.org/dist/DR-TarantoolQueue/ +Description: client for tarantool's queue + The module contains sync and async (coro) driver for tarantool queue. + +Package: dr-tarantool-queue +Architecture: all +Section: database +Suggests: libdr-tarantoolqueue-perl +Depends: ${misc:Depends}, ${perl:Depends}, + tarantool (>= 1.4.8+20130110-1) +Homepage: https://github.com/mailru/tarantool-queue/ +Description: queue components for tarantool + The package contains init.lua and tarantool.cfg to build queue daemon + using tarantool. + diff --git a/debian/copyright b/debian/copyright index 50eaa73c..c4d5dc41 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,29 +1,24 @@ -Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Debianized-By: Roman Tsisyk -Upstream-Name: tarantool-queue -Upstream-Contact: support@tarantool.org -Source: https://github.com/tarantool/queue +Format-Specification: http://anonscm.debian.org/viewvc/dep/web/deps/dep5.mdwn?view=markup&pathrev=135 +Maintainer: Dmitry E. Oboukhov +Source: http://search.cpan.org/dist/DR-TarantoolQueue/ +Name: DR-TarantoolQueue Files: * -Copyright: 2014-2016 Tarantool AUTHORS -License: BSD-2-Clause - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. +Copyright: 2012, Dmitry E. Oboukhov +License: Artistic or GPL-1+ + +License: Artistic + This program is free software; you can redistribute it and/or modify + it under the terms of the Artistic License, which comes with Perl. + . + On Debian systems, the complete text of the Artistic License can be + found in `/usr/share/common-licenses/Artistic'. + +License: GPL-1+ + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 1, or (at your option) + any later version. . - THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - SUCH DAMAGE. + On Debian systems, the complete text of version 1 of the GNU General + Public License can be found in `/usr/share/common-licenses/GPL-1'. diff --git a/debian/dr-tarantool-queue.dirs b/debian/dr-tarantool-queue.dirs new file mode 100644 index 00000000..7ab87635 --- /dev/null +++ b/debian/dr-tarantool-queue.dirs @@ -0,0 +1,3 @@ +/usr/lib/dr-tarantool-queue/ +/etc/tarantool/instances.available/ + diff --git a/debian/dr-tarantool-queue.install b/debian/dr-tarantool-queue.install new file mode 100644 index 00000000..ce824ad6 --- /dev/null +++ b/debian/dr-tarantool-queue.install @@ -0,0 +1,2 @@ +init.lua /usr/lib/dr-tarantool-queue/ +queue.cfg /etc/tarantool/instances.available/ diff --git a/debian/docs b/debian/libdr-tarantoolqueue-perl.docs similarity index 100% rename from debian/docs rename to debian/libdr-tarantoolqueue-perl.docs diff --git a/debian/libdr-tarantoolqueue-perl.install b/debian/libdr-tarantoolqueue-perl.install new file mode 100644 index 00000000..dd95e40e --- /dev/null +++ b/debian/libdr-tarantoolqueue-perl.install @@ -0,0 +1 @@ +debian/tmp/* diff --git a/debian/prebuild.sh b/debian/prebuild.sh deleted file mode 100755 index cab2e3ea..00000000 --- a/debian/prebuild.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -set -exu # Strict shell (w/o -o pipefail) - -# At the time of adding the changes, tarantool 1.10 is absent in the -# repositories Ubuntu impish and jammy. -if [[ $DIST == "impish" ]] || [[ $DIST == "jammy" ]]; then - curl -LsSf https://www.tarantool.io/release/2/installer.sh | sudo bash -else - curl -LsSf https://www.tarantool.io/release/1.10/installer.sh | sudo bash -fi diff --git a/debian/rules b/debian/rules index 2d33f6ac..c77b2a32 100755 --- a/debian/rules +++ b/debian/rules @@ -1,4 +1,33 @@ #!/usr/bin/make -f +vdeb = $(shell dpkg-parsechangelog \ + | grep ^Version|awk '{print $$2}'|sed 's/-[[:digit:]]\+$$//') +vmod = $(shell grep '^our[[:space:]]\+$$VERSION' lib/DR/TarantoolQueue.pm \ + |head -n 1 |awk '{print $$4}'|sed "s/[';']//g" ) +pkt = $(shell dpkg-parsechangelog|grep ^Source|awk '{print $$2}') + + %: dh $@ + +override_dh_install: + install -m 0644 tarantool.cfg queue.cfg + echo >> queue.cfg + echo script_dir = /usr/lib/dr-tarantool-queue/ >> queue.cfg + dh_install + make tardist + +override_dh_clean: + rm -f queue.cfg + rm -f DR-TarantoolQueue-?.??.tar.gz + dh_clean + +tarball: clean + test $(vdeb) = $(vmod) + test -d ../$(pkt)-$(vdeb) + cd .. && tar --exclude=debian --exclude=.git \ + -czvf $(pkt)_$(vdeb).orig.tar.gz \ + $(pkt)-$(vdeb) + +manifest: clean + find -type f|grep -v '\.git'|sed 's/^..//'|sort -u > MANIFEST diff --git a/debian/source/format b/debian/source/format index 163aaf8d..89ae9db8 100644 --- a/debian/source/format +++ b/debian/source/format @@ -1 +1 @@ -3.0 (quilt) +3.0 (native) diff --git a/debian/tarantool-queue.install b/debian/tarantool-queue.install deleted file mode 100644 index 141bda7a..00000000 --- a/debian/tarantool-queue.install +++ /dev/null @@ -1,3 +0,0 @@ -queue/*.lua usr/share/tarantool/queue/ -queue/abstract/*.lua usr/share/tarantool/queue/abstract/ -queue/abstract/driver/*.lua usr/share/tarantool/queue/abstract/driver/ diff --git a/debian/watch b/debian/watch new file mode 100644 index 00000000..5ef8708c --- /dev/null +++ b/debian/watch @@ -0,0 +1,2 @@ +version=3 +http://search.cpan.org/dist/DR-TarantoolQueue/ .*/DR-TarantoolQueue-v?(\d[\d.-]+)\.(?:tar(?:\.gz|\.bz2)?|tgz|zip)$ diff --git a/init.lua b/init.lua new file mode 100644 index 00000000..457910e4 --- /dev/null +++ b/init.lua @@ -0,0 +1,1271 @@ +-- vim: set ft=lua et : + +-- The library is free software; you can redistribute it and/or modify it +-- under the terms of either: + +-- a) the GNU General Public License as published by the Free Software +-- Foundation; either version 1, or (at your option) +-- any later version (http://dev.perl.org/licenses/gpl1.html) + +-- b) the "Artistic License" (http://dev.perl.org/licenses/artistic.html) + +-- This library is free software; you can redistribute it and/or modify +-- it under the same terms as Perl itself (Artistic or GPL-1+). + + +-- queue in tarantool + +-- tarantool config example: + + +-- space = [ +-- { +-- enabled = 1, +-- index = [ +-- { +-- type = "TREE", +-- unique = 1, +-- key_field = [ +-- { +-- fieldno = 0, +-- type = "STR" +-- } +-- ] +-- }, +-- { +-- type = "TREE", +-- unique = 0, +-- key_field = [ +-- { +-- fieldno = 1, # tube +-- type = "STR" +-- }, +-- { +-- fieldno = 2, # status +-- type = "STR" +-- }, +-- { +-- fieldno = 4, # ipri +-- type = "STR" +-- }, +-- { +-- fieldno = 5 # pri +-- type = "STR" +-- } +-- ] +-- }, +-- { +-- type = "TREE", +-- unique = 0, +-- key_field = [ +-- { +-- fieldno = 1, # tube +-- type = "STR" +-- }, +-- { +-- fieldno = 3, # next_event +-- type = "NUM64" +-- } +-- ] +-- }, +-- ] +-- } +-- ] + +-- If You want use method queue.put_unique you have to add additional +-- (fourth) index: +-- { +-- type = "TREE", +-- unique = 0, +-- key_field = [ +-- { +-- fieldno = 1, # tube +-- type = "STR" +-- }, +-- { +-- fieldno = 2, # status +-- type = "STR" +-- }, +-- { +-- fieldno = 12, # task data +-- type = "STR" +-- } +-- ] +-- } + +-- Glossary +-- space - number of space contains tubes +-- tube - queue name +-- ttl - time to live +-- ttr - time to release (when task is run) +-- delay - delay period for task + +(function(box) + +local FIBERS_PER_TUBE = 1 + +-- tuple structure +local i_uuid = 0 +local i_tube = 1 +local i_status = 2 +local i_event = 3 +local i_ipri = 4 +local i_pri = 5 +local i_cid = 6 +local i_created = 7 +local i_ttl = 8 +local i_ttr = 9 + +local i_cbury = 10 +local i_ctaken = 11 +local i_task = 12 + +local max_pri = 0xFF +local min_pri = 0 +local med_pri = 0x7F +local function pri_pack(pri) return box.pack('b', pri) end +local function pri_unpack(pri) return box.unpack('b', pri) end + +-- indexes +local idx_task = 0 +local idx_tube = 1 +local idx_event = 2 +local idx_data = 3 + +-- task statuses +local ST_READY = 'r' +local ST_DELAYED = 'd' +local ST_TAKEN = 't' +local ST_BURIED = 'b' +local ST_DONE = '*' + +-- timeout infinity +local TIMEOUT_INFINITY = 365 * 86400 + + +local human_status = {} + human_status[ST_READY] = 'ready' + human_status[ST_DELAYED] = 'delayed' + human_status[ST_TAKEN] = 'taken' + human_status[ST_BURIED] = 'buried' + human_status[ST_DONE] = 'done' + +local all_statuses = { + ST_READY, + ST_DELAYED, + ST_TAKEN, + ST_BURIED, + ST_DONE, +} + + +local function get_ipri(space, tube, inc) + local toptask + if inc < 0 then + toptask = box.select_limit(space, idx_tube, 0, 1, tube, ST_READY) + if toptask == nil then + return queue.default.ipri + end + else + toptask = box.select_reverse_range(space, idx_tube, 1, tube, ST_READY) + if toptask == nil then + return queue.default.ipri + end + if toptask[i_status] ~= ST_READY then + return queue.default.ipri + end + end + + local ipri = pri_unpack(toptask[i_ipri]) + if inc < 0 then + if ipri > min_pri - inc then + ipri = ipri + inc + else + ipri = min_pri + end + else + if ipri < max_pri - inc then + ipri = ipri + inc + else + ipri = max_pri + end + end + + return ipri +end + +local function to_time64(time) + return time * 1000000 +end + +local function from_time64(time) + return tonumber(time) / 1000000 +end + +local function rettask(task) + if task == nil then + return + end + + return task + :transform(i_event, i_task - i_event) + :transform(i_status, 1, human_status[ task[i_status] ]) +end + + +local function process_tube(space, tube) + + while true do + local now = box.time64() + local next_event = 3600 + while true do + local task = box.select_limit(space, idx_event, 0, 1, tube) + + if task == nil then + break + end + + local event = box.unpack('l', task[i_event]) + if event > now then + next_event = from_time64(event - now) + break + end + + local created = box.unpack('l', task[i_created]) + local ttl = box.unpack('l', task[i_ttl]) + local ttr = box.unpack('l', task[i_ttr]) + + if now >= created + ttl then + queue.stat[space][tube]:inc('ttl.total') + queue.stat[space][tube]:inc( + 'ttl.' .. human_status[task[i_status]]) + box.delete(space, task[i_uuid]) + + -- delayed -> ready + elseif task[i_status] == ST_DELAYED then + box.update(space, task[i_uuid], + '=p=p', + + i_status, + ST_READY, + + i_event, + created + ttl + ) + queue.consumers[space][tube]:put(true, 0) + -- taken -> ready + elseif task[i_status] == ST_TAKEN then + box.update(space, task[i_uuid], + '=p=p=p', + + i_status, + ST_READY, + + i_cid, + 0, + + i_event, + created + ttl + ) + queue.consumers[space][tube]:put(true, 0) + else + print("Internal error: unexpected task status: ", + task[i_status], + " (", human_status[ task[i_status] ], ")") + box.update(space, task[i_uuid], + '=p', + i_event, + now + to_time64(5) + ) + end + now = box.time64() + end + + queue.workers[space][tube].ch:get(next_event) + end +end + +local function consumer_dead_tube(space, tube, cid) + local index = box.space[space].index[idx_tube] + + for task in index:iterator(box.index.EQ, tube, ST_TAKEN) do + local created = box.unpack('l', task[i_created]) + local ttl = box.unpack('l', task[i_ttl]) + if box.unpack('i', task[i_cid]) == cid then + queue.stat[space][tube]:inc('ready_by_disconnect') + box.update( + space, + task[i_uuid], + '=p=p=p', + + i_status, + ST_READY, + + i_event, + created + ttl, + + i_cid, + 0 + ) + + + queue.consumers[space][tube]:put(true, 0) + queue.workers[space][tube].ch:put(true, 0) + end + end +end + + +local function consumer_dead(cid) + for space, tbt in pairs(queue.consumers) do + for tube, ch in pairs(tbt) do + consumer_dead_tube(space, tube, cid) + end + end +end + + + +if queue == nil then + queue = {} + queue.consumers = {} + queue.workers = {} + queue.stat = {} + queue.restart = {} + + setmetatable(queue.consumers, { + __index = function(tbs, space) + local spt = {} + setmetatable(spt, { + __index = function(tbt, tube) + local channel = box.ipc.channel(1) + rawset(tbt, tube, channel) + queue.restart_check(space, tube) + return channel + end + }) + rawset(tbs, space, spt) + return spt + end, + __gc = function(tbs) + for space, tubes in pairs(tbs) do + for tube, tbt in pairs(tubes) do + rawset(tubes, tube, nil) + end + rawset(tbs, space, nil) + end + end + } + ) + + setmetatable(queue.stat, { + __index = function(tbs, space) + local spt = {} + setmetatable(spt, { + __index = function(tbt, tube) + local stat = { + inc = function(t, cnt) + t[cnt] = t[cnt] + 1 + return t[cnt] + end + } + setmetatable(stat, { + __index = function(t, cnt) + rawset(t, cnt, 0) + return 0 + end + + }) + + rawset(tbt, tube, stat) + return stat + end + }) + rawset(tbs, space, spt) + return spt + end, + __gc = function(tbs) + for space, tubes in pairs(tbs) do + for tube, tbt in pairs(tubes) do + rawset(tubes, tube, nil) + end + rawset(tbs, space, nil) + end + end + } + ) + + setmetatable(queue.workers, { + __index = function(tbs, space) + + local spt = rawget(tbs, space) + spt = {} + setmetatable(spt, { + __index = function(tbt, tube) + + local v = rawget(tbt, tube) + + v = { fibers = {}, ch = box.ipc.channel(1) } + + -- rawset have to be before start fiber + rawset(tbt, tube, v) + + queue.restart_check(space, tube) + + for i = 1, FIBERS_PER_TUBE do + local fiber = box.fiber.create( + function() + box.fiber.detach() + process_tube(space, tube) + end + ) + box.fiber.resume(fiber) + table.insert(v.fibers, fiber) + end + + return v + end + }) + rawset(tbs, space, spt) + return spt + end, + + __gc = function(tbs) + for space, tubes in pairs(tbs) do + for tube, tbt in pairs(tubes) do + for i, fiber in pairs(tbt.fibers) do + box.fiber.cancel(fiber) + end + tbt.fibers = nil + tbt.ch = nil + rawset(tubes, tube, nil) + end + rawset(tbs, space, nil) + end + end + } + ) +end + +queue.default = {} + queue.default.pri = med_pri + queue.default.ipri = med_pri + queue.default.ttl = 3600 * 24 * 25 + queue.default.ttr = 60 + queue.default.delay = 0 + + + +queue.restart_check = function(space, tube) + space = tonumber(space) + if queue.restart[space] ~= nil and queue.restart[space][tube] then + return 'already started' + end + + if queue.restart[space] == nil then + queue.restart[space] = {} + end + + if queue.restart[space][tube] == nil then + queue.restart[space][tube] = true + end + + + local fiber = box.fiber.create( + function() + box.fiber.detach() + local wakeup = false + local index = box.space[space].index[idx_tube] + for task in index:iterator(box.index.EQ, tube, ST_TAKEN) do + local now = box.time64() + local event = box.unpack('l', task[i_event]) + local ttr = box.unpack('l', task[i_ttr]) + local ttl = box.unpack('l', task[i_ttl]) + local created = box.unpack('l', task[i_created]) + + + -- find task that is taken before tarantool + if event - ttr <= now - to_time64(box.info.uptime) + or not box.session.exists(box.unpack('i', task[i_cid])) + then + print( + 'task id=', task[i_uuid], + ' (space=', space, ', tube=', tube, ') ', + 'will be ready by restarting tarantool' + ) + queue.stat[space][tube]:inc('ready_by_restart') + box.update( + space, + task[i_tube], + '=p=p=p', + i_status, + ST_READY, + + i_event, + created + ttl, + + i_cid, + 0 + ) + wakeup = true + end + end + if wakeup then + queue.consumers[space][tube]:put(true, 0) + queue.workers[space][tube].ch:put(true, 0) + end + end + ) + box.fiber.resume(fiber) + return 'starting' +end + + +local function put_statistics(stat, space, tube) + if space == nil then + return + end + if tube == nil then + return + end + + local st = rawget(queue.stat, space) + if st == nil then + return + end + + st = rawget(st, tube) + if st == nil then + return + end + for name, value in pairs(st) do + if type(value) ~= 'function' then + + table.insert(stat, + 'space' .. tostring(space) .. '.' .. tostring(tube) + .. '.' .. tostring(name) + ) + table.insert(stat, tostring(value)) + end + + end + table.insert(stat, + 'space' .. tostring(space) .. '.' .. tostring(tube) + .. '.tasks.total' + ) + table.insert(stat, + tostring(box.space[space].index[idx_tube]:count(tube)) + ) + + for i, s in pairs(all_statuses) do + table.insert(stat, + 'space' .. tostring(space) .. '.' .. tostring(tube) + .. '.tasks.' .. human_status[s] + ) + table.insert(stat, + tostring( + box.space[space].index[idx_tube]:count(tube, s) + ) + ) + end +end + + +-- queue.statistics +-- returns statistics about all queues +queue.statistics = function( space, tube ) + + local stat = {} + + if space ~= nil and tube ~= nil then + space = tonumber(space) + put_statistics(stat, space, tube) + elseif space ~= nil then + space = tonumber(space) + local spt = rawget(queue.stat, space) + if spt ~= nil then + for tube, st in pairs(spt) do + put_statistics(stat, space, tube) + end + end + else + for space, spt in pairs(queue.stat) do + for tube, st in pairs(spt) do + put_statistics(stat, space, tube) + end + end + end + return stat + +end + + +local function put_task(space, tube, ipri, delay, ttl, ttr, pri, ...) + + ttl = tonumber(ttl) + if ttl <= 0 then + ttl = queue.default.ttl + end + ttl = to_time64(ttl) + + delay = tonumber(delay) + if delay <= 0 then + delay = queue.default.delay + end + delay = to_time64(delay) + ttl = ttl + delay + + + ttr = tonumber(ttr) + if ttr <= 0 then + ttr = queue.default.ttr + end + ttr = to_time64(ttr) + + pri = tonumber(pri) + pri = pri + queue.default.pri + if pri > max_pri then + pri = max_pri + elseif pri < min_pri then + pri = min_pri + end + + pri = max_pri - (pri - min_pri) + + local task + local now = box.time64() + + if delay > 0 then + task = box.insert(space, + box.uuid_hex(), + tube, + ST_DELAYED, + box.pack('l', now + delay), + pri_pack(ipri), + pri_pack(pri), + box.pack('i', 0), + box.pack('l', now), + box.pack('l', ttl), + box.pack('l', ttr), + box.pack('l', 0), + box.pack('l', 0), + ... + ) + else + task = box.insert(space, + box.uuid_hex(), + tube, + ST_READY, + box.pack('l', now + ttl), + pri_pack(ipri), + pri_pack(pri), + box.pack('i', 0), + box.pack('l', now), + box.pack('l', ttl), + box.pack('l', ttr), + box.pack('l', 0), + box.pack('l', 0), + ... + ) + queue.consumers[space][tube]:put(true, 0) + end + + queue.workers[space][tube].ch:put(true, 0) + + return rettask(task) + +end + + +-- queue.put(space, tube, delay, ttl, ttr, pri, ...) +-- put task into queue. +-- arguments +-- 1. tube - queue name +-- 2. delay - delay before task can be taken +-- 3. ttl - time to live (if delay > 0 ttl := ttl + delay) +-- 4. ttr - time to release (when task is taken) +-- 5. pri - priority +-- 6. ... - task data +queue.put = function(space, tube, ...) + space = tonumber(space) + queue.stat[space][tube]:inc('put') + return put_task(space, tube, queue.default.ipri, ...) +end + + +-- queue.put_unique(space, tube, delay, ttl, ttr, pri, ...) +-- put unique task into queue. +-- arguments +-- 1. tube - queue name +-- 2. delay - delay before task can be taken +-- 3. ttl - time to live (if delay > 0 ttl := ttl + delay) +-- 4. ttr - time to release (when task is taken) +-- 5. pri - priority +-- 6. ... - task data +queue.put_unique = function(space, tube, delay, ttl, ttr, pri, data, ...) + space = tonumber(space) + + delay = tonumber(delay) + if delay <= 0 then + delay = queue.default.delay + end + delay = to_time64(delay) + + local check_status = ST_READY + if delay > 0 then + check_status = ST_DELAYED + end + + if data == nil then + error('Can not put unique task without data') + end + + if box.space[space].index[idx_data] == nil then + error("Tarantool have to be configured to use queue.put_unique method") + end + + local task = box.select( space, idx_data, tube, check_status, data ) + if task ~= nil then + return rettask( task ) + end + queue.stat[space][tube]:inc('put') + return put_task(space, tube, queue.default.ipri, delay, ttl, ttr, pri, data, ...) +end + + +-- queue.urgent(space, tube, delay, ttl, ttr, pri, ...) +-- like queue.put but put task at begin of queue +queue.urgent = function(space, tube, delayed, ...) + space = tonumber(space) + delayed = tonumber(delayed) + queue.stat[space][tube]:inc('urgent') + + -- TODO: may decrease ipri before put_task + if delayed > 0 then + return put_task(space, tube, queue.default.ipri, delayed, ...) + end + + local ipri = get_ipri(space, tube, -1) + return put_task(space, tube, ipri, delayed, ...) +end + + +-- queue.take(space, tube, timeout) +-- take task for processing +queue.take = function(space, tube, timeout) + + space = tonumber(space) + + if timeout == nil then + timeout = TIMEOUT_INFINITY + else + timeout = tonumber(timeout) + if timeout < 0 then + error("Timeout can't be less then 0") + end + end + + local created = box.time() + + while true do + + local iterator = box.space[space].index[idx_tube] + :iterator(box.index.EQ, tube, ST_READY) + + for task in iterator do + local now = box.time64() + local created = box.unpack('l', task[i_created]) + local ttr = box.unpack('l', task[i_ttr]) + local ttl = box.unpack('l', task[i_ttl]) + local event = now + ttr + if event > created + ttl then + event = created + ttl + -- tube started too late + if event <= now then + break + end + end + + + task = box.update(space, + task[i_uuid], + '=p=p=p+p', + i_status, + ST_TAKEN, + + i_event, + event, + + i_cid, + box.session.id(), + + i_ctaken, + 1 + ) + + queue.workers[space][tube].ch:put(true, 0) + queue.consumers[space][tube]:put(true, 0) + queue.stat[space][tube]:inc('take') + return rettask(task) + end + + if timeout == 0 then + queue.stat[space][tube]:inc('take_timeout') + return + end + + if timeout > 0 then + local now = box.time() + if now < created + timeout then + queue.consumers[space][tube]:get(created + timeout - now) + else + queue.stat[space][tube]:inc('take_timeout') + return + end + else + queue.consumers[space][tube]:get() + end + end +end + + +-- queue.truncate(space, tube) +queue.truncate = function(space, tube) + space = tonumber(space) + + local index = box.space[space].index[idx_tube] + local task_ids = {} + + for task in index:iterator(box.index.EQ, tube) do + table.insert(task_ids, task[i_uuid]) + end + + for _, task_id in pairs(task_ids) do + box.space[space]:delete(task_id) + end + + return #task_ids +end + + +-- queue.delete(space, id) +-- deletes task from queue +queue.delete = function(space, id) + space = tonumber(space) + local task = box.select(space, idx_task, id) + if task == nil then + error("Task not found") + end + + queue.stat[space][ task[i_tube] ]:inc('delete') + return rettask(box.delete(space, id)) +end + + +-- queue.ack(space, id) +-- done task processing (task will be deleted) +queue.ack = function(space, id) + space = tonumber(space) + local task = box.select(space, idx_task, id) + if task == nil then + error('Task not found') + end + + if task[i_status] ~= ST_TAKEN then + error('Task is not taken') + end + + if box.unpack('i', task[i_cid]) ~= box.session.id() then + error('Only consumer that took the task can it ack') + end + + queue.stat[space][ task[i_tube] ]:inc('ack') + return rettask(box.delete(space, id)) +end + + +-- queue.touch(space, id) +-- prolong ttr for taken task +queue.touch = function(space, id) + space = tonumber(space) + local task = box.select(space, idx_task, id) + if task == nil then + error('Task not found') + end + + if task[i_status] ~= ST_TAKEN then + error('Task is not taken') + end + + if box.unpack('i', task[i_cid]) ~= box.session.id() then + error('Only consumer that took the task can it touch') + end + + local ttr = box.unpack('l', task[i_ttr]) + local ttl = box.unpack('l', task[i_ttl]) + local created = box.unpack('l', task[i_created]) + local now = box.time64() + + local event + + if created + ttl > now + ttr then + event = now + ttr + else + event = created + ttl + end + + task = box.update(space, id, '=p', i_event, event) + + queue.stat[space][ task[i_tube] ]:inc('touch') + return rettask(task) +end + + +-- queue.done(space, id, ...) +-- marks task as done, replaces task's data +queue.done = function(space, id, ...) + space = tonumber(space) + local task = box.select(space, 0, id) + if task == nil then + error("Task not found") + end + if task[i_status] ~= ST_TAKEN then + error('Task is not taken') + end + + if box.unpack('i', task[i_cid]) ~= box.session.id() then + error('Only consumer that took the task can it done') + end + + local event = box.unpack('l', task[i_created]) + + box.unpack('l', task[i_ttl]) + local tube = task[i_tube] + + task = task + :transform(i_task, #task, ...) + :transform(i_status, 1, ST_DONE) + :transform(i_event, 1, event) + + task = box.replace(space, task:unpack()) + queue.workers[space][tube].ch:put(true, 0) + queue.stat[space][ tube ]:inc('done') + return rettask(task) +end + + +-- queue.bury(space, id) +-- bury task that is taken +queue.bury = function(space, id) + space = tonumber(space) + local task = box.select(space, 0, id) + if task == nil then + error("Task not found") + end + if task[i_status] ~= ST_TAKEN then + error('Task is not taken') + end + + if box.unpack('i', task[i_cid]) ~= box.session.id() then + error('Only consumer that took the task can it done') + end + + local event = box.unpack('l', task[i_created]) + + box.unpack('l', task[i_ttl]) + local tube = task[i_tube] + + task = box.update(space, task[i_uuid], + '=p=p+p', + + i_status, + ST_BURIED, + + i_event, + event, + + i_cbury, + 1 + ) + + queue.workers[space][tube].ch:put(true, 0) + queue.stat[space][ tube ]:inc('bury') + return rettask(task) +end + + +-- queue.dig(space, id) +-- dig(unbury) task +queue.dig = function(space, id) + + space = tonumber(space) + local task = box.select(space, 0, id) + if task == nil then + error("Task not found") + end + if task[i_status] ~= ST_BURIED then + error('Task is not buried') + end + + local tube = task[i_tube] + + task = box.update(space, task[i_uuid], + '=p+p', + + i_status, + ST_READY, + + i_cbury, + 1 + ) + + queue.workers[space][tube].ch:put(true, 0) + queue.stat[space][ tube ]:inc('dig') + return rettask(task) +end + +queue.unbury = queue.dig + + +-- queue.kick(space, tube, count) +queue.kick = function(space, tube, count) + space = tonumber(space) + + local index = box.space[space].index[idx_tube] + + if count == nil then + count = 1 + end + count = tonumber(count) + + if count <= 0 then + return 0 + end + + local kicked = 0 + + for task in index:iterator(box.index.EQ, tube, ST_BURIED) do + box.update(space, task[i_uuid], + '=p+p', + + i_status, + ST_READY, + + i_cbury, + 1 + ) + kicked = kicked + 1 + queue.stat[space][ tube ]:inc('dig') + end + + return kicked +end + + +-- queue.release(space, id [, delay [, ttl ] ]) +-- marks task as ready (or delayed) +queue.release = function(space, id, delay, ttl) + space = tonumber(space) + local task = box.select(space, idx_task, id) + if task == nil then + error('Task not found') + end + if task[i_status] ~= ST_TAKEN then + error('Task is not taken') + end + if box.unpack('i', task[i_cid]) ~= box.session.id() then + error('Only consumer that took the task can it release') + end + + local tube = task[i_tube] + + local now = box.time64() + local created = box.unpack('l', task[i_created]) + + if delay == nil then + delay = 0 + else + delay = to_time64(tonumber(delay)) + if delay <= 0 then + delay = 0 + end + end + + if ttl == nil then + ttl = box.unpack('l', task[i_ttl]) + else + ttl = tonumber(ttl) + if ttl > 0 then + ttl = to_time64(ttl) + ttl = ttl + now - created + if delay > 0 then + ttl = ttl + delay + end + else + ttl = box.unpack('l', task[i_ttl]) + end + end + + + + + if delay > 0 then + local event = now + delay + if event > created + ttl then + event = created + ttl + end + + task = box.update(space, + id, + '=p=p=p=p', + + i_status, + ST_DELAYED, + + i_event, + event, + + i_ttl, + ttl, + + i_cid, + 0 + ) + else + task = box.update(space, + id, + '=p=p=p=p', + + i_status, + ST_READY, + + i_event, + created + ttl, + + i_ttl, + ttl, + + i_cid, + 0 + ) + queue.consumers[space][tube]:put(true, 0) + end + queue.workers[space][tube].ch:put(true, 0) + + + queue.stat[space][ task[i_tube] ]:inc('release') + + return rettask(task) +end + + +-- queue.requeue(space, id) +-- marks task as ready and push it at end of queue +queue.requeue = function(space, id) + space = tonumber(space) + local task = box.select(space, idx_task, id) + if task == nil then + error('Task not found') + end + if task[i_status] ~= ST_TAKEN then + error('Task is not taken') + end + if box.unpack('i', task[i_cid]) ~= box.session.id() then + error('Only consumer that took the task can it release') + end + + local tube = task[i_tube] + + local now = box.time64() + + + local ipri = get_ipri(space, tube, 1) + + local created = box.unpack('l', task[i_created]) + local ttl = box.unpack('l', task[i_ttl]) + + + task = box.update(space, + id, + '=p=p=p=p', + + i_status, + ST_READY, + + i_event, + created + ttl, + + i_cid, + 0, + + i_ipri, + pri_pack(ipri) + ) + queue.consumers[space][tube]:put(true, 0) + queue.workers[space][tube].ch:put(true, 0) + + queue.stat[space][ task[i_tube] ]:inc('requeue') + + return rettask(task) +end + + +-- queue.meta(space, id) +-- metainformation about task +-- returns: +-- 1. uuid:str +-- 2. tube:str +-- 3. status:str +-- 4. event:time64 +-- 5. ipri:num(str) +-- 6. pri:num(str) +-- 7. cid:num +-- 8. created:time64 +-- 9. ttl:time64 +-- 10. ttr:time64 +-- 11. cbury:num +-- 12. ctaken:num +-- 13. now:time64 +queue.meta = function(space, id) + space = tonumber(space) + local task = box.select(space, 0, id) + if task == nil then + error('Task not found'); + end + + queue.stat[space][ task[i_tube] ]:inc('meta') + + task = task + :transform(i_pri, 1, tostring(pri_unpack(task[i_pri]))) + :transform(i_ipri, 1, tostring(pri_unpack(task[i_ipri]))) + :transform(i_task, #task - i_task, box.time64()) + :transform(i_status, 1, human_status[ task[i_status] ]) + + local meta_ttl = box.unpack('l', task[i_ttl]) + if meta_ttl ~= nil then + task = task:transform(i_ttl, 1, box.pack('l', from_time64(meta_ttl))) + end + + local meta_ttr = box.unpack('l', task[i_ttr]) + if meta_ttr ~= nil then + task = task:transform(i_ttr, 1, box.pack('l', from_time64(meta_ttr))) + end + return task +end + + +-- queue.peek(space, id) +-- peek task +queue.peek = function(space, id) + space = tonumber(space) + local task = box.select(space, 0, id) + if task == nil then + error("Task not found") + end + + queue.stat[space][ task[i_tube] ]:inc('peek') + return rettask(task) +end + +box.session.on_disconnect( function() consumer_dead(box.session.id()) end ) + + +end)(box) diff --git a/lib/DR/TarantoolQueue.pm b/lib/DR/TarantoolQueue.pm new file mode 100644 index 00000000..efb4b570 --- /dev/null +++ b/lib/DR/TarantoolQueue.pm @@ -0,0 +1,702 @@ +package DR::TarantoolQueue; +use DR::Tarantool (); +use utf8; +use strict; +use warnings; +use Mouse; +use Carp; +use JSON::XS; +require DR::TarantoolQueue::Task; +$Carp::Internal{ (__PACKAGE__) }++; + +our $VERSION = '0.27'; + +=head1 NAME + +DR::TarantoolQueue - client for tarantool's queue + + +=head1 SYNOPSIS + + my $queue = DR::TarantoolQueue->new( + host => 'tarantool.host', + port => 33014, + tube => 'request_queue', + space => 11, + + connect_opts => { # see perldoc DR::Tarantool + reconnect_period => 1, + reconnect_always => 1 + } + ); + + + # put empty task into queue with name 'request_queue' + my $task = $queue->put; + + my $task = $queue->put(data => [ 1, 2, 3 ]); + + printf "task.id = %s\n", $task->id; + +=head2 DESCRIPTION + +The module contains sync and async (coro) driver for tarantool queue. + +=head1 ATTRIBUTES + +=head2 host (ro) & port (ro) + +Tarantool's parameters. + +=head2 connect_opts (ro) + +Additional options for L. HashRef. + +=head2 coro (ro) + +If B (default) the driver will use L tarantool's driver, +otherwise the driver will use sync driver. + +=head2 ttl (rw) + +Default B for tasks. + +=head2 ttr (rw) + +Default B for tasks. + +=head2 pri (rw) + +Default B for tasks. + +=head2 delay (rw) + +Default B for tasks. + +=head2 space (rw) + +Default B for tasks. + +=head2 tube (rw) + +Default B for tasks. + + +=head2 defaults + +Defaults for queues. B. Key is tube name. Value is a hash with +the following fields: + +=over + +=item ttl + +=item ttr + +=item delay + +=item pri + +=back + +Methods L (L) use these parameters if they +are absent (otherwise it uses the same global attributes). + +=cut + +with 'DR::TarantoolQueue::JSE'; + +has host => (is => 'ro', isa => 'Maybe[Str]'); +has port => (is => 'ro', isa => 'Maybe[Str]'); +has coro => (is => 'ro', isa => 'Bool', default => 1); + +has ttl => (is => 'rw', isa => 'Maybe[Num]'); +has ttr => (is => 'rw', isa => 'Maybe[Num]'); +has pri => (is => 'rw', isa => 'Maybe[Num]'); +has delay => (is => 'rw', isa => 'Maybe[Num]'); +has space => (is => 'rw', isa => 'Maybe[Str]'); +has tube => (is => 'rw', isa => 'Maybe[Str]'); +has connect_opts => (is => 'ro', isa => 'HashRef', default => sub {{}}); + +has defaults => (is => 'ro', isa => 'HashRef', default => sub {{}}); + +has tnt => ( + is => 'rw', + isa => 'Object', + lazy => 1, + builder => sub { + my ($self) = @_; + unless ($self->coro) { + if (DR::Tarantool->can('rsync_tarantool')) { + return DR::Tarantool::rsync_tarantool + port => $self->port, + host => $self->host, + spaces => {}, + %{ $self->connect_opts } + ; + } else { + return DR::Tarantool::tarantool + port => $self->port, + host => $self->host, + spaces => {}, + %{ $self->connect_opts } + ; + } + } + + require Coro; + if ($self->{tnt_waiter}) { + push @{ $self->{tnt_waiter} } => $Coro::current; + Coro::schedule(); + return $self->tnt; + } + + $self->{tnt_waiter} = []; + my $tnt = DR::Tarantool::coro_tarantool + port => $self->port, + host => $self->host, + spaces => {}, + %{ $self->connect_opts } + ; + $_->ready for @{ $self->{tnt_waiter} }; + delete $self->{tnt_waiter}; + return $tnt; + } +); + + +sub _check_opts($@) { + my $h = shift; + my %can = map { ($_ => 1) } @_; + + for (keys %$h) { + next if $can{$_}; + croak 'unknown option: ' . $_; + } +} + +sub _producer { + my ($self, $method, $o) = @_; + + _check_opts $o, qw(space tube delay ttl ttr pri data); + + my $space = $o->{space}; + $space = $self->space unless defined $space; + croak 'space was not defined' unless defined $space; + + my $tube = $o->{tube}; + $tube = $self->tube unless defined $tube; + croak 'tube was not defined' unless defined $tube; + + my ($ttl, $ttr, $pri, $delay); + + for ([\$ttl, 'ttl'], [\$delay, 'delay'], [\$ttr, 'ttr'], [\$pri, 'pri']) { + my $rv = $_->[0]; + my $n = $_->[1]; + + if (exists $o->{$n}) { + $$rv = $o->{$n}; + } else { + if (exists $self->defaults->{ $tube }) { + if (exists $self->defaults->{ $tube }{ $n }) { + $$rv = $self->defaults->{ $tube }{ $n }; + } else { + $$rv = $self->$n; + } + } else { + $$rv = $self->$n; + } + } + $$rv ||= 0; + + } + + + my $tuple = $self->tnt->call_lua( + "queue.$method" => [ + $space, + $tube, + $delay, + $ttl, + $ttr, + $pri, + $self->jse->encode($o->{data}) + ] + ); + + return DR::TarantoolQueue::Task->tuple($tuple, $space, $self); +} + +=head1 METHODS + +=head2 new + + my $q = DR::TarantoolQueue->new(host => 'abc.com', port => 123); + +Creates new queue(s) accessor. + +=cut + +=head2 dig + + $q->dig(task => $task); + $task->dig; # the same + + $q->dig(id => $task->id); + $q->dig(id => $task->id, space => $task->space); + +'Dig up' a buried task. Checks, that the task is buried. +The task status is changed to ready. + +=head2 unbury + +Is a synonym of L. + + +=head2 delete + + $q->delete(task => $task); + $task->delete; # the same + + $q->delete(id => $task->id); + $q->delete(id => $task->id, space => $task->space); + +Delete a task from the queue (regardless of task state or status). + +=head2 peek + + $q->peek(task => $task); + $task->peek; # the same + + $q->peek(id => $task->id); + $q->peek(id => $task->id, space => $task->space); + +Return a task by task id. + + +=head2 statistics + + my $s = $q->statistics; + my $s = $q->statistics(space => 123); + my $s = $q->statistics(space => 123, tube => 'abc'); + my $s = DR::TarantoolQueue->statistics(space => 123); + my $s = DR::TarantoolQueue->statistics(space => 123, tube => 'abc'); + +Return queue module statistics, since server start. +The statistics is broken down by queue id. +Only queues on which there was some activity are +included in the output. + + +=cut + +sub statistics { + my ($self, %o) = @_; + _check_opts \%o, qw(space tube); + unless (exists $o{space}) { + $o{space} = $self->space if ref $self; + } + unless (exists $o{tube}) { + $o{tube} = $self->tube if ref $self; + } + + croak 'space was not defined' + if defined $o{tube} and !defined $o{space}; + + my $raw = $self->tnt->call_lua( + "queue.statistics" => [ + defined($o{space}) ? $o{space} : (), + defined($o{tube}) ? $o{tube} : () + ] + )->raw; + return { @$raw }; +} + + +=head2 get_meta + +Task was processed (and will be deleted after the call). + + my $m = $q->get_meta(task => $task); + my $m = $q->get_meta(id => $task->id); + +Returns a hashref with fields: + + +=over + +=item id + +task id + +=item tube + +queue id + +=item status + +task status + +=item event + +time of the next important event in task life time, for example, +when ttl or ttr expires, in microseconds since start of the UNIX epoch. + +=item ipri + +internal value of the task priority + +=item pri + +task priority as set when the task was added to the queue + +=item cid + +consumer id, of the consumer which took the task (only if the task is taken) + +=item created + +time when the task was created (microseconds since start of the UNIX epoch) + +=item ttl + +task time to live (microseconds) + +=item ttr + +task time to run (microseconds) + +=item cbury + +how many times the task was buried + +=item ctaken + +how many times the task was taken + +=item now + +time recorded when the meta was called + +=back + +=cut + +sub get_meta { + my ($self, %o) = @_; + _check_opts \%o, qw(task id space); + croak 'task was not defined' unless $o{task} or $o{id}; + + my ($id, $space, $tube); + if ($o{task}) { + ($id, $space, $tube) = ($o{task}->id, + $o{task}->space, $o{task}->tube); + } else { + ($id, $space, $tube) = @o{'id', 'space', 'tube'}; + $space = $self->space unless defined $o{space}; + croak 'space is not defined' unless defined $space; + $tube = $self->tube unless defined $tube; + } + + + my $fields = [ + { name => 'id', type => 'STR' }, + { name => 'tube', type => 'STR' }, + { name => 'status', type => 'STR' }, + { name => 'event', type => 'NUM64' }, + { name => 'ipri', type => 'STR', }, + { name => 'pri', type => 'STR', }, + { name => 'cid', type => 'NUM', }, + { name => 'created', type => 'NUM64', }, + { name => 'ttl', type => 'NUM64' }, + { name => 'ttr', type => 'NUM64' }, + { name => 'cbury', type => 'NUM' }, + { name => 'ctaken', type => 'NUM' }, + { name => 'now', type => 'NUM64' }, + ]; + my $tuple = $self->tnt->call_lua( + "queue.meta" => [ $space, $id ], fields => $fields + )->raw; + + + return { map { ( $fields->[$_]{name}, $tuple->[$_] ) } 0 .. $#$fields }; +} + + + + +=head1 Producer methods + +=head2 put + + $q->put; + $q->put(data => { 1 => 2 }); + $q->put(space => 1, tube => 'abc', + delay => 10, ttl => 3600, + ttr => 60, pri => 10, data => [ 3, 4, 5 ]); + $q->put(data => 'string'); + + +Enqueue a task. Returns new L object. +The list of fields with task data (C<< data => ... >>) is optional. + + +If 'B' and (or) 'B' aren't defined the method +will try to use them from L object. + +=cut + +sub put { + my ($self, %opts) = @_; + return $self->_producer(put => \%opts); +} + +=head2 put_unique + + $q->put_unique(data => { 1 => 2 }); + $q->put_unique(space => 1, tube => 'abc', + delay => 10, ttl => 3600, + ttr => 60, pri => 10, data => [ 3, 4, 5 ]); + $q->put_unique(data => 'string'); + + +Enqueue an unique task. Returns new L object, +if it was not enqueued previously. Otherwise it will return existing task. +The list of fields with task data (C<< data => ... >>) is optional. + + +If 'B' and (or) 'B' aren't defined the method +will try to use them from L object. + +=cut + +sub put_unique { + my ($self, %opts) = @_; + return $self->_producer(put_unique => \%opts); +} + +=head2 urgent + +Enqueue a task. The task will get the highest priority. +If delay is not zero, the function is equivalent to +L. + +=cut + +sub urgent { + my ($self, %opts) = @_; + return $self->_producer(urgent => \%opts); +} + + +=head1 Consumer methods + +=head2 take + + my $task = $q->take; + my $task = $q->take(timeout => 0.5); + my $task = $q->take(space => 1, tube => 'requests, timeout => 20); + +If there are tasks in the queue ready for execution, +take the highest-priority task. Otherwise, wait for +a ready task to appear in the queue, and, as soon as +it appears, mark it as taken and return to the consumer. +If there is a timeout, and the task doesn't appear until +the timeout expires, returns B. If timeout is not +given, waits indefinitely. + +All the time while the consumer is working on a task, +it must keep the connection to the server open. If a +connection disappears while the consumer is still +working on a task, the task is put back on the ready list. + +=cut + +sub take { + my ($self, %o) = @_; + _check_opts \%o, qw(space tube timeout); + $o{space} = $self->space unless defined $o{space}; + croak 'space was not defined' unless defined $o{space}; + $o{tube} = $self->tube unless defined $o{tube}; + croak 'tube was not defined' unless defined $o{tube}; + $o{timeout} ||= 0; + + + my $tuple = $self->tnt->call_lua( + 'queue.take' => [ + $o{space}, + $o{tube}, + $o{timeout} + ] + ); + + + return DR::TarantoolQueue::Task->tuple($tuple, $o{space}, $self); +} + + +=head2 ack + + $q->ack(task => $task); + $task->ack; # the same + + $q->ack(id => $task->id); + $q->ack(space => $task->space, id => $task->id); + + +Confirm completion of a task. Before marking a task as +complete, this function verifies that: + +=over + +=item * + +the task is taken + +=item * + +the consumer that is confirming the task is the one which took it + +=back + +Consumer identity is established using a session identifier. +In other words, the task must be confirmed by the same connection +which took it. If verification fails, the function returns an error. + +On success, deletes the task from the queue. Throws an exception otherwise. + + +=head2 requeue + + $q->requeue(task => $task); + $task->requeue; # the same + + $q->requeue(id => $task->id); + $q->requeue(id => $task->id, space => $task->space); + +Return a task to the queue, the task is not executed. +Puts the task at the end of the queue, so that it's executed +only after all existing tasks in the queue are executed. + + +=head2 bury + + $q->bury(task => $task); + $task->bury; # the same + + $q->bury(id => $task->id); + $q->bury(id => $task->id, space => $task->space); + +Mark a task as B. This special status excludes the task +from the active list, until it's dug up. This function is useful +when several attempts to execute a task lead to a failure. Buried +tasks can be monitored by the queue owner, and treated specially. + + +=cut + +for my $m (qw(ack requeue bury dig unbury delete peek)) { + no strict 'refs'; + next if *{ __PACKAGE__ . "::$m" }{CODE}; + *{ __PACKAGE__ . "::$m" } = sub { + my ($self, %o) = @_; + _check_opts \%o, qw(task id space); + croak 'task was not defined' unless $o{task} or $o{id}; + + my ($id, $space); + if ($o{task}) { + ($id, $space) = ($o{task}->id, $o{task}->space); + } else { + ($id, $space) = @o{'id', 'space'}; + $space = $self->space unless defined $o{space}; + croak 'space is not defined' unless defined $space; + } + + my $tuple = $self->tnt->call_lua( "queue.$m" => [ $space, $id ] ); + my $task = DR::TarantoolQueue::Task->tuple($tuple, $space, $self); + + if ($m eq 'delete') { + $task->_set_status('removed'); + } elsif ($m eq 'ack') { + $task->_set_status('ack(removed)'); + } + $task; + } +} + + +=head2 release + + $q->release(task => $task); + $task->release; # the same + + $q->release(id => $task->id, space => $task->space); + $q->release(task => $task, delay => 10); # delay the task + $q->release(task => $task, ttl => 3600); # append task's ttl + +Return a task back to the queue: the task is not executed. +Additionally, a new time to live and re-execution delay can be provided. + +=cut + +sub release { + my ($self, %o) = @_; + _check_opts \%o, qw(task id space ttl delay); + $o{delay} ||= 0; + my ($id, $space); + if ($o{task}) { + ($id, $space) = ($o{task}->id, $o{task}->space); + } else { + ($id, $space) = @o{'id', 'space'}; + $space = $self->space unless defined $o{space}; + croak 'space is not defined' unless defined $space; + } + my $tuple = $self->tnt->call_lua('queue.release' => + [ $space, $id, $o{delay}, $o{ttl} || () ] + ); + return DR::TarantoolQueue::Task->tuple($tuple, $space, $self); +} + + + +=head2 done + + $q->done(task => $task, data => { result => '123' }); + $task->done(data => { result => '123' }); # the same + $q->done(id => $task->id, space => $task->space); + +Mark a task as complete (done), but don't delete it. Replaces task +data with the supplied B. + +=cut + +sub done { + my ($self, %o) = @_; + _check_opts \%o, qw(task id space data); + my ($id, $space); + if ($o{task}) { + ($id, $space) = ($o{task}->id, $o{task}->space); + } else { + ($id, $space) = @o{'id', 'space'}; + $space = $self->space unless defined $o{space}; + croak 'space is not defined' unless defined $space; + } + my $tuple = $self->tnt->call_lua('queue.done' => + [ $space, $id, $self->jse->encode($o{data}) ] + ); + return DR::TarantoolQueue::Task->tuple($tuple, $space, $self); +} + + +=head1 COPYRIGHT AND LICENCE + + Copyright (C) 2012 by Dmitry E. Oboukhov + Copyright (C) 2012 by Roman V. Nikolaev + +This library is free software; you can redistribute it and/or modify +it under the same terms as Perl itself, either Perl version 5.8.8 or, +at your option, any later version of Perl 5 you may have available. + +=cut + +__PACKAGE__->meta->make_immutable(); diff --git a/lib/DR/TarantoolQueue/JSE.pm b/lib/DR/TarantoolQueue/JSE.pm new file mode 100644 index 00000000..5f7e4cbb --- /dev/null +++ b/lib/DR/TarantoolQueue/JSE.pm @@ -0,0 +1,19 @@ +package DR::TarantoolQueue::JSE; +use Mouse::Role; +use utf8; +use strict; +use warnings; + + +has jse => ( + is => 'ro', + isa => 'Object', + lazy => 1, + builder => '_build_jse' +); + +sub _build_jse { + return JSON::XS->new->allow_nonref->utf8; +} + +1; diff --git a/lib/DR/TarantoolQueue/Task.pm b/lib/DR/TarantoolQueue/Task.pm new file mode 100644 index 00000000..22933eb0 --- /dev/null +++ b/lib/DR/TarantoolQueue/Task.pm @@ -0,0 +1,105 @@ +package DR::TarantoolQueue::Task; +use utf8; +use strict; +use warnings; +use Mouse; +use JSON::XS (); +use Carp; + +has space => (is => 'ro', isa => 'Str', required => 1); +has status => ( + is => 'ro', + isa => 'Str', + required => 1, + writer => '_set_status' +); +has tube => (is => 'ro', isa => 'Str', required => 1); +has id => (is => 'ro', isa => 'Str', required => 1); +has rawdata => ( + is => 'ro', + isa => 'Str|Undef', + required => 1, + writer => '_set_rawdata', + trigger => sub { $_[0]->_clean_data } +); +has data => ( + is => 'ro', + isa => 'HashRef|ArrayRef|Str|Undef', + lazy => 1, + builder => '_build_data', + clearer => '_clean_data' +); + +has queue => (is => 'ro', isa => 'Object|Undef', weak_ref => 1); + +with 'DR::TarantoolQueue::JSE'; + + +$Carp::Internal{ (__PACKAGE__) }++; + +sub _build_data { + my ($self) = @_; + return undef unless defined $self->rawdata; + + my $res = eval { $self->jse->decode( $self->rawdata ) }; + warn $@ if $@; + return $res; +} + + +for my $m (qw(ack requeue bury dig unbury delete peek)) { + no strict 'refs'; + next if *{ __PACKAGE__ . "::$m" }{CODE}; + *{ __PACKAGE__ . "::$m" } = sub { + my ($self) = @_; + croak "Can't find queue for task" unless $self->queue; + my $task = $self->queue->$m(task => $self); + $self->_set_status($task->status); + $self; + } +} + +for my $m (qw(get_meta)) { + no strict 'refs'; + next if *{ __PACKAGE__ . "::$m" }{CODE}; + *{ __PACKAGE__ . "::$m" } = sub { + my ($self) = @_; + croak "Can't find queue for task" unless $self->queue; + return $self->queue->$m(task => $self); + } +} + + +sub tuple { + my ($class, $tuple, $space, $queue) = @_; + return undef unless $tuple; + my $raw = $tuple->raw; + return $class->new( + id => $raw->[0], + tube => $raw->[1], + status => $raw->[2], + rawdata => $raw->[3], + space => $space, + queue => $queue, + ); +} + + +sub done { + my ($self, %o) = @_; + $o{data} = $self->data unless exists $o{data}; + my $task = $self->queue->done(task => $self, %o); + $self->_set_status( $task->status ); + $self->_set_rawdata( $task->rawdata ); + $self; +} + +sub release { + my ($self, %o) = @_; + my $task = $self->queue->release(task => $self, %o); + $self->_set_status( $task->status ); + $self; +} + + +__PACKAGE__->meta->make_immutable(); diff --git a/lib/DR/TarantoolQueue/Worker.pm b/lib/DR/TarantoolQueue/Worker.pm new file mode 100644 index 00000000..ddaea1ae --- /dev/null +++ b/lib/DR/TarantoolQueue/Worker.pm @@ -0,0 +1,364 @@ +use utf8; +use strict; +use warnings; +package DR::TarantoolQueue::Worker; +use Carp; +use Mouse; +use Coro; +use Data::Dumper; +use Encode qw(encode_utf8); + +=head1 NAME + +DR::TarantoolQueue::Worker - template for workers + +=head1 SYNOPSIS + + my $worker = DR::TarantoolQueue::Worker->new( + count => 10, # defaults 1 + queue => $queue + ); + + sub process { + my ($task) = @_; + + + ... do something with task + + + } + + $worker->run(\&process) + +=head1 DESCRIPTION + +=over + +=item * + +Process function can throw exception. The task will be buried (if process +function didn't change task status yet. + +=item * + +If process function didn't change task status (didn't call B, or +L) worker calls +L. + +=item * + +L method catches B and B and waits for all process +functions are done and then do return. + +=item * + +Worker uses default B and B in queue. So You have to define +them in Your queue or here. + +=back + +=head1 ATTRIBUTES + +=cut + +=head2 count + +Count of process functions that can do something at the same time. +Default value is B<1>. The attribute means something if Your B +function uses L and Your queue uses L, too. + +=cut + +has count => isa => 'Num', is => 'rw', default => 1; + + +=head2 queue + +Ref to Your queue. + +=cut + +has queue => isa => 'DR::TarantoolQueue', is => 'ro', required => 1; + +=head2 space & tube + +Space and tube for processing queue. + +=cut + +has space => isa => 'Str|Undef', is => 'ro'; +has tube => isa => 'Str|Undef', is => 'ro'; + +=head2 restart + +The function will be called if L is reached. + +=cut + +has restart => isa => 'CodeRef|Undef', is => 'rw'; + +=head2 restart_limit + +How many tasks can be processed before restart worker. + +If B is 0, restart mechanizm will be disabled. + +If L callback isn't defined, restart mechanizm will be disabled. + +Each processed task increments common taskcounter. When B is +reached by the counter, worker don't take new task and call L +function. After L worker will continue to process tasks. + +In L callback user can do L or L +to avoid memory leaks. + + DR::TarantoolQueue::Worker->new( + restart_limit => 100, + restart => sub { exec perl => $0 }, + queue => $q, + count => 10 + )->run(sub { ... }); + +=cut + +has restart_limit => isa => 'Num', is => 'rw', default => 0; + +=head1 PRIVATE ATTRIBUTES + +=head2 timeout + +timeout for queue.take + +=cut + +has timeout => isa => 'Num', is => 'ro', default => 2; + +=head2 is_run + +B means that workers are run + +=cut + +has is_run => isa => 'Bool', is => 'rw', default => 0; + +=head2 is_stopping + +B means that workers are stopping (by B/B/L) + +=cut + +has is_stopping => isa => 'Bool', is => 'rw', default => 0; + + +has stop_waiters => isa => 'ArrayRef', is => 'ro', default => sub {[]}; + + +has mailto => isa => 'Maybe[Str]', is => 'ro'; +has mailfrom => isa => 'Maybe[Str]', is => 'ro'; +has mailsublect => isa => 'Str', is => 'ro', default => 'Worker died'; +has mailheaders => isa => 'HashRef[Str]', is => 'ro', default => sub {{}}; + +has restart_check => isa => 'CodeRef', is => 'ro', default => sub {sub { 0 }}; + +=head1 METHODS + +=head2 run(CODEREF[, CODEREF]) + +Run workers. Two arguments: + +=over + +=item process function + +Function will receive three arguments: + +=over + +=item task + +=item queue + +=item task number + +=back + +=item debug function + +The function can be used to show internal debug messages. + +=over + +=item * + +Debug messages aren't finished by B (C<\n>). + +=item * + +The function will be called as L. + +=back + +=back + +=cut + +sub run { + my ($self, $cb, $debugf) = @_; + croak 'process subroutine is not CODEREF' unless 'CODE' eq ref $cb; + $debugf = sub { } + unless defined $debugf; + croak 'debugf subroutine is not CODEREF' unless 'CODE' eq ref $debugf; + + croak 'worker is already run' if $self->is_run; + + local $SIG{TERM} = sub { + $debugf->('SIGTERM was received, stopping...'); + $self->is_stopping( 1 ) + }; + local $SIG{INT} = sub { + $debugf->('SIGINT was received, stopping...'); + $self->is_stopping( 1 ) + }; + + + $self->is_run( 1 ); + $self->is_stopping( 0 ); + + my $no; + my @f; + while(1) { + ($no, @f) = (0); + + for (1 .. $self->count) { + push @f => async { + while($self->is_run and !$self->is_stopping) { + last if $self->restart and $no >= $self->restart_limit; + last if $self->restart and $self->restart_check->(); + my $task = $self->queue->take( + defined($self->space) ? (space => $self->space) : (), + defined($self->tube) ? (tube => $self->tube) : (), + timeout => $self->timeout, + ); + next unless $task; + + $no++; + eval { + $cb->( $task, $self->queue, $no ); + }; + + if ($@) { + my $err = $@; + $debugf->('Worker was died (%s)', $@); + eval { + $self->sendmail( + $task, + sprintf "Worker was died: %s", $err + ); + }; + if ($@) { + $debugf->("Can't send mail (%s)", $@); + } + if ($task->status eq 'taken') { + eval { $task->bury }; + if ($@) { + $debugf->("Can't bury task %s: %s", + $task->id, $@); + } + } + next; + } + if ($task->status eq 'taken') { + eval { $task->ack }; + if ($@) { + $debugf->("Can't ack task %s: %s", $task->id, $@); + } + next; + } + } + } + } + + $_->join for @f; + + last unless $self->is_run; + last if $self->is_stopping; + last unless $self->restart; + last unless $no >= $self->restart_limit; + $self->restart->( ); + } + + $self->is_run( 0 ); + $self->is_stopping( 0 ); + while(@{ $self->stop_waiters }) { + my $w = shift @{ $self->stop_waiters }; + $w->ready; + } + return $self->count; +} + + +=head2 sendmail + +Send mail about worker crash + +=cut + +sub sendmail { + my ($self, $task, $error) = @_; + return unless $self->mailto; + return unless $self->mailfrom; + + my $subject = encode_utf8 $self->mailsublect; + + require MIME::Lite; + require MIME::Words; + + $subject .= sprintf' (space: %s, tube: %s)', $task->space, $task->tube; + $subject = MIME::Words::encode_mimeword($subject, 'B', 'utf-8'); + + my $mail = MIME::Lite->new( + From => $self->mailfrom || 'dimka@uvw.ru', + To => $self->mailto || 'dimka@uvw.ru', + Subject => $subject, + Type => 'multipart/fixed', + ); + + local $Data::Dumper::Indent = 1; + local $Data::Dumper::Terse = 1; + local $Data::Dumper::Useqq = 1; + local $Data::Dumper::Deepcopy = 1; + local $Data::Dumper::Maxdepth = 0; + + + $mail->attach( + Type => 'text/plain; charset=utf-8', + Data => encode_utf8($error), + ); + $mail->attach( + Type => 'text/plain; charset=utf-8', + Filename => 'task.dump.txt', + Disposition => 'inline', + Data => Dumper($task), + ); + + $mail->add($_ => $self->mailheaders->{$_}) for keys %{ $self->mailheaders }; + $mail->send; +} + +=head2 stop + +Stop worker cycle + +=cut + +sub stop { + my ($self) = @_; + return 0 unless $self->is_run; + $self->is_stopping( 1 ); + push @{ $self->stop_waiters } => $Coro::current; + Coro::schedule; + return $self->is_run; +} + +__PACKAGE__->meta->make_immutable(); + diff --git a/queue-scm-1.rockspec b/queue-scm-1.rockspec deleted file mode 100644 index 73122a96..00000000 --- a/queue-scm-1.rockspec +++ /dev/null @@ -1,35 +0,0 @@ -package = 'queue' -version = 'scm-1' -source = { - url = 'git+https://github.com/tarantool/queue.git', - branch = 'master', -} -description = { - summary = "A set of persistent in-memory queues", - homepage = 'https://github.com/tarantool/queue.git', - license = 'BSD', -} -dependencies = { - 'lua >= 5.1' -} -build = { - type = 'builtin', - - modules = { - ['queue.abstract'] = 'queue/abstract.lua', - ['queue.abstract.state'] = 'queue/abstract/state.lua', - ['queue.abstract.queue_session'] = 'queue/abstract/queue_session.lua', - ['queue.abstract.queue_state'] = 'queue/abstract/queue_state.lua', - ['queue.abstract.driver.fifottl'] = 'queue/abstract/driver/fifottl.lua', - ['queue.abstract.driver.utubettl'] = 'queue/abstract/driver/utubettl.lua', - ['queue.abstract.driver.fifo'] = 'queue/abstract/driver/fifo.lua', - ['queue.abstract.driver.utube'] = 'queue/abstract/driver/utube.lua', - ['queue.abstract.driver.limfifottl'] = 'queue/abstract/driver/limfifottl.lua', - ['queue.compat'] = 'queue/compat.lua', - ['queue.util'] = 'queue/util.lua', - ['queue'] = 'queue/init.lua', - ['queue.version'] = 'queue/version.lua' - } -} - --- vim: syntax=lua diff --git a/queue/CMakeLists.txt b/queue/CMakeLists.txt deleted file mode 100644 index 9234cb66..00000000 --- a/queue/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ -install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/init.lua - DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/) -install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/compat.lua - DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/) -install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/util.lua - DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/) -install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/abstract.lua - DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/) -install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/version.lua - DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/) -install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/abstract/state.lua - DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/abstract) -install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/abstract/queue_session.lua - DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/abstract) -install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/abstract/queue_state.lua - DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/abstract) -install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/abstract/driver/fifo.lua - DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/abstract/driver/) -install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/abstract/driver/utube.lua - DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/abstract/driver/) -install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/abstract/driver/fifottl.lua - DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/abstract/driver/) -install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/abstract/driver/utubettl.lua - DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/abstract/driver/) -install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/abstract/driver/limfifottl.lua - DESTINATION ${TARANTOOL_INSTALL_LUADIR}/${PROJECT_NAME}/abstract/driver/) diff --git a/queue/abstract.lua b/queue/abstract.lua deleted file mode 100644 index 9c2d4342..00000000 --- a/queue/abstract.lua +++ /dev/null @@ -1,979 +0,0 @@ -local log = require('log') -local fiber = require('fiber') -local uuid = require('uuid') - -local session = require('queue.abstract.queue_session') -local state = require('queue.abstract.state') -local queue_state = require('queue.abstract.queue_state') - -local util = require('queue.util') -local qc = require('queue.compat') -local num_type = qc.num_type -local str_type = qc.str_type - --- since: --- https://github.com/tarantool/tarantool/commit/f266559b167b4643c377724eebde9651877d6cc9 -local ddl_txn_supported = qc.check_version({2, 2, 1}) - --- The term "queue session" has been added to the queue. One "queue session" --- can include many connections (box.session). For clarity, the box.session --- will be referred to as connection below. -local connection = box.session - -local queue = { - tube = setmetatable({}, { - __call = function(self, tube_name) - if tube_name and type(tube_name) ~= 'string' then - error('argument #1 should be string or nil') - end - -- return all names of tubes - if tube_name == nil then - local rv = {} - for name, _ in pairs(self) do - table.insert(rv, name) - end - return rv - end - -- return true/false if tube name is provided - return not (self[tube_name] == nil) - end - }), - stat = {} -} - --- Release tasks that don't have session_uuid stored in inactive sessions. -local function tube_release_all_orphaned_tasks(tube) - local prefix = ('queue: [tube "%s"] '):format(tube.name) - - -- We lean on stable iterators in this function. - -- https://github.com/tarantool/tarantool/issues/1796 - if not qc.check_version({1, 7, 5}) then - log.error(prefix .. 'no stable iterator support: skip task releasing') - log.error(prefix .. 'some tasks may stuck in taken state perpetually') - log.error(prefix .. 'update tarantool to >= 1.7.5 or take the risk') - return - end - - log.info(prefix .. 'releasing all taken task (may take a while)') - local released = 0 - for _, task in tube.raw:tasks_by_state(state.TAKEN) do - local taken = box.space._queue_taken_2.index.task:get{ - tube.tube_id, task[1] - } - if taken and session.exist_shared(taken[4]) then - log.info(prefix .. - ('skipping task: %d, tube_id: %d'):format(task[1], - tube.tube_id)) - else - tube.raw:release(task[1], {}) - released = released + 1 - end - end - log.info(prefix .. ('released %d tasks'):format(released)) -end - ---- Check whether the task has been taken in a current session or in a session --- with session uuid = session_uuid. --- Throw an error if task is not take in the session. -local function check_task_is_taken(tube_id, task_id, session_uuid) - local _taken = box.space._queue_taken_2.index.task:get{tube_id, task_id} - - session_uuid = session_uuid or session.identify(connection.id()) - if _taken == nil or _taken[4] ~= session_uuid then - error("Task was not taken") - end -end - --- tube methods -local tube = {} - --- This check must be called from all public tube methods. -local function check_state(method) - if queue_state.get() ~= queue_state.states.RUNNING then - log.error(('%s: queue is in %s state'):format(method, queue_state.show())) - return false - end - - return true -end - -function tube.put(self, data, opts) - if not check_state("put") then - return nil - end - opts = opts or {} - local task = self.raw:put(data, opts) - return self.raw:normalize_task(task) -end - -local conds = {} -local releasing_connections = {} - -function tube.take(self, timeout) - if not check_state("take") then - return nil - end - timeout = util.time(timeout or util.TIMEOUT_INFINITY) - local task = self.raw:take() - if task ~= nil then - return self.raw:normalize_task(task) - end - - while timeout > 0 do - local started = fiber.time64() - local time = util.event_time(timeout) - local tid = self.tube_id - local fid = fiber.id() - local conn_id = connection.id() - - box.space._queue_consumers:insert{conn_id, fid, tid, time, started} - conds[fid] = qc.waiter() - conds[fid]:wait(tonumber(timeout) / 1000000) - conds[fid]:free() - box.space._queue_consumers:delete{conn_id, fid} - - -- We don't take a task if the connection is in a - -- disconnecting state. - if releasing_connections[fid] then - releasing_connections[fid] = nil - return nil - end - - task = self.raw:take() - - if task ~= nil then - return self.raw:normalize_task(task) - end - - local elapsed = fiber.time64() - started - timeout = timeout > elapsed and timeout - elapsed or 0 - end -end - -function tube.touch(self, id, delta) - if not check_state("touch") then - return - end - if delta == nil then - return - end - if delta < 0 then -- if delta is lesser then 0, then it's zero - delta = 0 - elseif delta > util.MAX_TIMEOUT then -- no ttl/ttr for this task - delta = util.TIMEOUT_INFINITY - else -- convert to usec - delta = delta * 1000000 - end - if delta == 0 then - return - end - - check_task_is_taken(self.tube_id, id) - - local space_name = box.space._queue:get{self.name}[3] - queue.stat[space_name]:inc('touch') - - return self.raw:normalize_task(self.raw:touch(id, delta)) -end - -function tube.ack(self, id) - if not check_state("ack") then - return nil - end - check_task_is_taken(self.tube_id, id) - local tube = box.space._queue:get{self.name} - local space_name = tube[3] - - self:peek(id) - -- delete task - box.space._queue_taken_2.index.task:delete{self.tube_id, id} - local result = self.raw:normalize_task( - self.raw:delete(id):transform(2, 1, state.DONE) - ) - -- swap delete and ack call counters - queue.stat[space_name]:inc('ack') - queue.stat[space_name]:dec('delete') - return result -end - -local function tube_release_internal(self, id, opts, session_uuid) - opts = opts or {} - check_task_is_taken(self.tube_id, id, session_uuid) - - box.space._queue_taken_2.index.task:delete{self.tube_id, id} - self:peek(id) - return self.raw:normalize_task(self.raw:release(id, opts)) -end - -function tube.release(self, id, opts) - if not check_state("release") then - return nil - end - return tube_release_internal(self, id, opts) -end - --- Release all tasks. -function tube.release_all(self) - if not check_state("tube") then - return - end - local prefix = ('queue: [tube "%s"] '):format(self.name) - - log.info(prefix .. 'releasing all taken task (may take a while)') - local released = 0 - for _, task in self.raw:tasks_by_state(state.TAKEN) do - self.raw:release(task[1], {}) - released = released + 1 - end - log.info(('%s released %d tasks'):format(prefix, released)) -end - -function tube.peek(self, id) - if not check_state("peek") then - return nil - end - local task = self.raw:peek(id) - if task == nil then - error(("Task %s not found"):format(tostring(id))) - end - return self.raw:normalize_task(task) -end - -function tube.bury(self, id) - if not check_state("bury") then - return nil - end - local task = self:peek(id) - local is_taken, _ = pcall(check_task_is_taken, self.tube_id, id) - if is_taken then - box.space._queue_taken_2.index.task:delete{self.tube_id, id} - end - if task[2] == state.BURIED then - return task - end - return self.raw:normalize_task(self.raw:bury(id)) -end - -function tube.kick(self, count) - if not check_state("kick") then - return nil - end - count = count or 1 - return self.raw:kick(count) -end - -function tube.delete(self, id) - if not check_state("delete") then - return nil - end - self:peek(id) - return self.raw:normalize_task(self.raw:delete(id)) -end - --- drop tube -function tube.drop(self) - if not check_state("drop") then - return nil - end - local tube_name = self.name - - local tube = box.space._queue:get{tube_name} - if tube == nil then - error("Tube not found") - end - - local tube_id = tube[2] - - local cons = box.space._queue_consumers.index.consumer:min{tube_id} - - if cons ~= nil and cons[3] == tube_id then - error("There are consumers connected the tube") - end - - local taken = box.space._queue_taken_2.index.task:min{tube_id} - if taken ~= nil and taken[1] == tube_id then - error("There are taken tasks in the tube") - end - - if self.raw.drop ~= nil then - self.raw:drop() - else - local space_name = tube[3] - box.space[space_name]:drop() - end - box.space._queue:delete{tube_name} - -- drop queue - queue.tube[tube_name] = nil - -- drop statistics - queue.stat[tube_name] = nil - return true -end - --- truncate tube --- (delete everything from tube) -function tube.truncate(self) - if not check_state("truncate") then - return - end - self.raw:truncate() -end - -function tube.on_task_change(self, cb) - local old_cb = self.on_task_change_cb - self.on_task_change_cb = cb or (function() end) - return old_cb -end - -function tube.grant(self, user, args) - if not check_state("grant") then - return - end - local function tube_grant_space(user, name, tp) - box.schema.user.grant(user, tp or 'read,write', 'space', name, { - if_not_exists = true, - }) - end - - local function tube_grant_func(user, name) - box.schema.func.create(name, { if_not_exists = true }) - box.schema.user.grant(user, 'execute', 'function', name, { - if_not_exists = true - }) - end - - args = args or {} - - tube_grant_space(user, '_queue', 'read') - tube_grant_space(user, '_queue_consumers') - tube_grant_space(user, '_queue_taken_2') - self.raw:grant(user, {if_not_exists = true}) - session.grant(user) - - if args.call then - tube_grant_func(user, 'queue.identify') - tube_grant_func(user, 'queue.statistics') - local prefix = (args.prefix or 'queue.tube') .. ('.%s:'):format(self.name) - tube_grant_func(user, prefix .. 'put') - tube_grant_func(user, prefix .. 'take') - tube_grant_func(user, prefix .. 'touch') - tube_grant_func(user, prefix .. 'ack') - tube_grant_func(user, prefix .. 'release') - tube_grant_func(user, prefix .. 'peek') - tube_grant_func(user, prefix .. 'bury') - tube_grant_func(user, prefix .. 'kick') - tube_grant_func(user, prefix .. 'delete') - end - - if args.truncate then - local prefix = (args.prefix or 'queue.tube') .. ('.%s:'):format(self.name) - tube_grant_func(user, prefix .. 'truncate') - end - -end - --- methods -local method = {} - --- List of required driver methods. -local required_driver_methods = { - 'normalize_task', - 'put', - 'take', - 'delete', - 'release', - 'bury', - 'kick', - 'peek', - 'touch', - 'truncate', - 'tasks_by_state' -} - --- gh-126 Check the driver API. -local function check_driver_api(tube_impl, tube_type) - for _, v in pairs(required_driver_methods) do - if tube_impl[v] == nil then - error(string.format('The "%s" driver does not have an ' .. - 'implementation of method "%s".', tube_type, v)) - end - end -end - --- Cache of already verified drivers. -local checked_drivers = {} - -local function make_self(driver, space, tube_name, tube_type, tube_id, opts) - opts = opts or {} - local self - - -- wakeup consumer if queue have new task - local on_task_change = function(task, stats_data) - self.on_task_change_cb(task, stats_data) - - -- task was removed - if task == nil then return end - - -- We cannot use a local variable to access the space `_queue_taken_2` - -- because it can be recreated in `switch_in_replicaset()`. - local queue_consumers = box.space._queue_consumers - - -- if task was taken and become other state - local taken = box.space._queue_taken_2.index.task:get{tube_id, task[1]} - if taken ~= nil then - box.space._queue_taken_2:delete{taken[1], taken[2]} - end - -- task switched to ready (or new task) - if task[2] == state.READY then - local tube_id = self.tube_id - local consumer = queue_consumers.index.consumer:min{tube_id} - - if consumer ~= nil then - if consumer[3] == tube_id then - queue_consumers:delete{consumer[1], consumer[2]} - local cond = conds[consumer[2]] - if cond then - cond:signal(consumer[2]) - end - end - end - -- task switched to taken - register in taken space - elseif task[2] == state.TAKEN then - local conn_id = connection.id() - local session_uuid = session.identify(conn_id) - box.space._queue_taken_2:insert{ - self.tube_id, - task[1], - conn_id, - session_uuid, - fiber.time64() - } - end - if stats_data ~= nil then - queue.stat[space.name]:inc(stats_data) - end - if stats_data == 'delete' then - queue.stat[space.name]:inc('done') - end - end - - self = setmetatable({ - raw = driver.new(space, on_task_change, opts), - name = tube_name, - type = tube_type, - tube_id = tube_id, - opts = opts, - }, { - __index = tube - }) - - if checked_drivers[tube_type] == nil then - check_driver_api(self.raw, tube_type) - checked_drivers[tube_type] = true - end - - self:on_task_change(opts.on_task_change) - queue.tube[tube_name] = self - - return self -end - ---- Release all session tasks. -local function release_session_tasks(session_uuid) - local taken_tasks = box.space._queue_taken_2.index.uuid:select{session_uuid} - - for _, task in pairs(taken_tasks) do - local tube = box.space._queue.index.tube_id:get{task[1]} - if tube == nil then - log.error("Inconsistent queue state: tube %d not found", task[1]) - box.space._queue_taken_2.index.task:delete{task[1], task[2]} - else - log.warn("Session %s closed, release task %s(%s)", - uuid.frombin(session_uuid):str(), task[2], tube[1]) - tube_release_internal(queue.tube[tube[1]], task[2], nil, - session_uuid) - end - end -end - -function method.state() - return queue_state.show() -end - -function method._on_consumer_disconnect() - local conn_id = connection.id() - - -- wakeup all waiters - while true do - local waiter = box.space._queue_consumers.index.pk:min{conn_id} - if waiter == nil then - break - end - -- Don't touch the other consumers - if waiter[1] ~= conn_id then - break - end - box.space._queue_consumers:delete{waiter[1], waiter[2]} - local cond = conds[waiter[2]] - if cond then - releasing_connections[waiter[2]] = true - cond:signal(waiter[2]) - end - end - - session.disconnect(conn_id) -end - --- function takes tuples and recreates tube -local function recreate_tube(tube_tuple) - local name, id, space_name, tube_type, opts = tube_tuple:unpack() - - local driver = queue.driver[tube_type] - if driver == nil then - error("Unknown tube type " .. tostring(tube_type)) - end - - local space = box.space[space_name] - if space == nil then - error(("Space '%s' doesn't exists"):format(space_name)) - end - return make_self(driver, space, name, tube_type, id, opts) -end - --- Function takes new queue state. --- The "RUNNING" and "WAITING" states do not require additional actions. -local function on_state_change(state) - if state == queue_state.states.STARTUP then - local replicaset_mode = queue.cfg['in_replicaset'] or false - -- gh-202: In replicaset mode, tubes can be created and deleted on different nodes. - -- Accordingly, it is necessary to rebuild the queue.tube index. - if replicaset_mode then - for _, tube_name in pairs(queue.tube()) do - queue.tube[tube_name] = nil - end - for _, tube_tuple in box.space._queue:pairs() do - if queue.driver[tube_tuple[4]] ~= nil then - recreate_tube(tube_tuple) - end - end - end - for name, tube in pairs(queue.tube) do - tube_release_all_orphaned_tasks(tube) - log.info('queue: [tube "%s"] start driver', name) - if not tube.raw.start then - log.warn('queue: [tube "%s"] method start is not implemented', - tube.name) - else - tube.raw:start() - end - end - session.start() - elseif state == queue_state.states.ENDING then - for name, tube in pairs(queue.tube) do - log.info('queue: [tube "%s"] stop driver', name) - if not tube.raw.stop then - log.warn('queue: [tube "%s"] method stop is not implemented', - tube.name) - else - tube.raw:stop() - end - end - session.stop() - else - error('on_state_change: unexpected queue state') - end -end - -------------------------------------------------------------------------------- --- create tube -function method.create_tube(tube_name, tube_type, opts) - if not check_state("create_tube") then - return - end - opts = opts or {} - if opts.if_not_exists == nil then - opts.if_not_exists = false - end - if opts.if_not_exists == true and queue.tube[tube_name] ~= nil then - return queue.tube[tube_name] - end - - local replicaset_mode = queue.cfg['in_replicaset'] or false - if replicaset_mode and opts.temporary then - error("Cannot create temporary tube in replicaset mode") - end - - local driver = queue.driver[tube_type] - if driver == nil then - error("Unknown tube type " .. tostring(tube_type)) - end - -- space name must be equal to tube name - -- https://github.com/tarantool/queue/issues/9#issuecomment-83019109 - local space_name = tube_name - if box.space[space_name] ~= nil and opts.if_not_exists == false then - error(("Space '%s' already exists"):format(space_name)) - end - - -- if tube tuple was already presented, then recreate old tube - local ptube = box.space._queue:get{tube_name} - if ptube ~= nil then - local self = recreate_tube(ptube) - self:on_task_change(opts.on_task_change) - return self - end - - -- create tube space - local space = driver.create_space(space_name, opts) - - -- create tube record - local last = box.space._queue.index.tube_id:max() - local tube_id = 0 - if last ~= nil then - tube_id = last[2] + 1 - end - local self = make_self(driver, space, tube_name, tube_type, tube_id, opts) - opts.on_task_change = nil - box.space._queue:insert{tube_name, tube_id, space_name, tube_type, opts} - return self -end - --- --- Replicaset mode switch. --- --- Running a queue in master-replica mode requires that --- `_queue_taken_2` space was not temporary. --- When the queue is running in single mode, --- the space is converted to temporary mode to increase performance. --- -local function switch_in_replicaset(replicaset_mode) - if replicaset_mode == nil then - log.warn('queue: queue required after box.cfg{}') - replicaset_mode = false - end - - if not box.space._queue_taken_2 then - return - end - - if box.space._queue_taken_2.temporary and replicaset_mode == false then - return - end - - if not box.space._queue_taken_2.temporary and replicaset_mode == true then - return - end - - box.schema.create_space('_queue_taken_2_mgr', { - temporary = not replicaset_mode, - format = { - {name = 'tube_id', type = num_type()}, - {name = 'task_id', type = num_type()}, - {name = 'connection_id', type = num_type()}, - {name = 'session_uuid', type = str_type()}, - {name = 'taken_time', type = num_type()} - }}) - - box.space._queue_taken_2_mgr:create_index('task', { - type = 'tree', - parts = {1, num_type(), 2, num_type()}, - unique = true - }) - box.space._queue_taken_2_mgr:create_index('uuid', { - type = 'tree', - parts = {4, str_type()}, - unique = false - }) - - box.begin() -- Disable implicit yields until the transaction ends. - for _, tuple in box.space._queue_taken_2:pairs() do - box.space._queue_taken_2_mgr:insert(tuple) - end - - if ddl_txn_supported then - -- We can do it inside a transaction. - box.space._queue_taken_2:drop() - box.space._queue_taken_2_mgr:rename('_queue_taken_2') - end - - local status, err = pcall(box.commit) - if not status then - error(('Error migrate _queue_taken_2: %s'):format(tostring(err))) - end - - if not ddl_txn_supported then - -- Do it outside a transaction becase DDL does not support - -- multi-statement transactions. - box.space._queue_taken_2:drop() - box.space._queue_taken_2_mgr:rename('_queue_taken_2') - end -end - --- create or join infrastructure -function method.start() - -- tube_name, tube_id, space_name, tube_type, opts - local _queue = box.space._queue - if _queue == nil then - _queue = box.schema.create_space('_queue', { - format = { - {name = 'tube_name', type = str_type()}, - {name = 'tube_id', type = num_type()}, - {name = 'space_name', type = str_type()}, - {name = 'tube_type', type = str_type()}, - {name = 'opts', type = '*'} - } - }) - _queue:create_index('tube', { - type = 'tree', - parts = {1, str_type()}, - unique = true - }) - _queue:create_index('tube_id', { - type = 'tree', - parts = {2, num_type()}, - unique = true - }) - end - - local _cons = box.space._queue_consumers - if _cons == nil then - -- connection, fid, tube, time - _cons = box.schema.create_space('_queue_consumers', { - temporary = true, - format = { - {name = 'connection_id', type = num_type()}, - {name = 'fiber_id', type = num_type()}, - {name = 'tube_id', type = num_type()}, - {name = 'event_time', type = num_type()}, - {name = 'fiber_time', type = num_type()} - } - }) - _cons:create_index('pk', { - type = 'tree', - parts = {1, num_type(), 2, num_type()}, - unique = true - }) - _cons:create_index('consumer', { - type = 'tree', - parts = {3, num_type(), 4, num_type()}, - unique = false - }) - end - - -- Remove deprecated space - if box.space._queue_taken ~= nil then - box.space._queue_taken:drop() - end - - local replicaset_mode = queue.cfg['in_replicaset'] or false - if box.space._queue_taken_2 == nil then - -- tube_id, task_id, connection_id, session_uuid, time - box.schema.create_space('_queue_taken_2', { - temporary = not replicaset_mode, - format = { - {name = 'tube_id', type = num_type()}, - {name = 'task_id', type = num_type()}, - {name = 'connection_id', type = num_type()}, - {name = 'session_uuid', type = str_type()}, - {name = 'taken_time', type = num_type()} - }}) - - box.space._queue_taken_2:create_index('task', { - type = 'tree', - parts = {1, num_type(), 2, num_type()}, - unique = true - }) - box.space._queue_taken_2:create_index('uuid', { - type = 'tree', - parts = {4, str_type()}, - unique = false - }) - else - switch_in_replicaset(queue.cfg['in_replicaset']) - end - - for _, tube_tuple in _queue:pairs() do - -- Recreate tubes for registered drivers only. - -- Check if a driver exists for this type of tube. - if queue.driver[tube_tuple[4]] ~= nil then - local tube = recreate_tube(tube_tuple) - -- gh-66: release all taken tasks on start - tube_release_all_orphaned_tasks(tube) - end - end - - session.on_session_remove(release_session_tasks) - session.start() - - connection.on_disconnect(queue._on_consumer_disconnect) - queue_state.init(on_state_change) - return queue -end - ---- Register the custom driver. --- Unlike the "register_driver" method from init.lua, this method --- recreates the existing tubes of the registered driver. -function method.register_driver(driver_name, tube_ctr) - if type(tube_ctr.create_space) ~= 'function' or - type(tube_ctr.new) ~= 'function' then - error('tube control methods must contain functions "create_space"' - .. ' and "new"') - end - if queue.driver[driver_name] then - error(('overriding registered driver "%s"'):format(driver_name)) - end - queue.driver[driver_name] = tube_ctr - - -- Recreates the existing tubes of the registered driver. - local _queue = box.space._queue - for _, tube_tuple in _queue:pairs() do - if tube_tuple[4] == driver_name then - local tube = recreate_tube(tube_tuple) - -- Release all task for tube on start. - tube_release_all_orphaned_tasks(tube) - end - end -end - -local function build_stats(space) - local stats = {tasks = {}, calls = { - ack = 0, bury = 0, delete = 0, - kick = 0, put = 0, release = 0, - take = 0, touch = 0, - -- for *ttl queues only - ttl = 0, ttr = 0, delay = 0, - }} - - local st = rawget(queue.stat, space) or {} - local idx_tube = 1 - - -- add api calls stats - for name, value in pairs(st) do - if type(value) ~= 'function' and name ~= 'done' then - stats['calls'][name] = value - end - end - - local total = 0 - -- add tasks by state count - for i, s in pairs(state) do - local st = i:lower() - stats['tasks'][st] = box.space[space].index[idx_tube]:count(s) - total = total + stats['tasks'][st] - end - - -- add total tasks count - stats['tasks']['total'] = total - stats['tasks']['done'] = st.done or 0 - - return stats -end - ---- Identifies the connection and return the UUID of the current session. --- If session_uuid ~= nil: associate the connection with given session. -function method.identify(session_uuid) - return session.identify(connection.id(), session_uuid) -end - ---- Configure of the queue module. --- If an invalid value or an unknown option --- is used, an error will be thrown. -local function cfg(self, opts) - opts = opts or {} - local session_opts = {} - - -- Set default in_replicaset value. - if opts['in_replicaset'] == nil then - opts['in_replicaset'] = false - end - - -- Temporary spaces are not allowed in replicaset mode. - if opts['in_replicaset'] == true and box.space._queue then - local temp_tubes = "" - for _, tube in box.space._queue:pairs() do - if tube[5].temporary then - temp_tubes = temp_tubes .. ', ' .. tube[1] - end - end - - if #temp_tubes ~= 0 then - temp_tubes = temp_tubes:sub(3) -- Cut first ', '. - opts['in_replicaset'] = false - log.error('Queue: cannot set `in_replicaset = true`: ' - .. 'temporary tube(s) exists: ' .. temp_tubes) - end - end - - -- Check all options before configuring so that - -- the configuration is done transactionally. - for key, val in pairs(opts) do - if key == 'ttr' then - session_opts[key] = val - elseif key == 'in_replicaset' then - if type(val) ~= 'boolean' then - error('Invalid value of in_replicaset: ' .. tostring(val)) - end - session_opts[key] = val - else - error('Unknown option ' .. tostring(key)) - end - end - - switch_in_replicaset(opts['in_replicaset']) - - -- Configuring the queue_session module. - session.cfg(session_opts) - - for key, val in pairs(opts) do - self[key] = val - end -end - -queue.cfg = setmetatable({}, { __call = cfg }) - -queue.statistics = function(space) - if space ~= nil then - return build_stats(space) - end - - local stats = {} - for _, tube_rc in box.space._queue:pairs() do - local tube_name = tube_rc[1] - stats[tube_name] = build_stats(tube_name) - end - - return stats -end - -queue.stats = queue.statistics - -setmetatable(queue.stat, { - __index = function(tbs, space) - local spt = setmetatable({ - inc = function(t, cnt) - t[cnt] = t[cnt] + 1 - return t[cnt] - end, - dec = function(t, cnt) - t[cnt] = t[cnt] - 1 - return t[cnt] - end - }, { - __index = function(t, cnt) - rawset(t, cnt, 0) - return 0 - end - }) - rawset(tbs, space, spt) - return spt - end, - __gc = function(tbs) - for space, tubes in pairs(tbs) do - for tube, tbt in pairs(tubes) do - rawset(tubes, tube, nil) - end - rawset(tbs, space, nil) - end - end - } -) - -return setmetatable(queue, { __index = method }) diff --git a/queue/abstract/driver/fifo.lua b/queue/abstract/driver/fifo.lua deleted file mode 100644 index 1cb65f93..00000000 --- a/queue/abstract/driver/fifo.lua +++ /dev/null @@ -1,204 +0,0 @@ -local state = require('queue.abstract.state') - -local num_type = require('queue.compat').num_type -local str_type = require('queue.compat').str_type - -local tube = {} -local method = {} - --- validate space of queue -local function validate_space(space) - -- check indexes - local indexes = {'task_id', 'status'} - for _, index in pairs(indexes) do - if space.index[index] == nil then - error(string.format('space "%s" does not have "%s" index', - space.name, index)) - end - end -end - --- create space -function tube.create_space(space_name, opts) - local space_opts = {} - local if_not_exists = opts.if_not_exists or false - space_opts.temporary = opts.temporary or false - space_opts.engine = opts.engine or 'memtx' - space_opts.format = { - {name = 'task_id', type = num_type()}, - {name = 'status', type = str_type()}, - {name = 'data', type = '*'} - } - - local space = box.space[space_name] - if if_not_exists and space then - -- Validate the existing space. - validate_space(box.space[space_name]) - return space - end - - space = box.schema.create_space(space_name, space_opts) - space:create_index('task_id', { - type = 'tree', - parts = {1, num_type()} - }) - space:create_index('status', { - type = 'tree', - parts = {2, str_type(), 1, num_type()} - }) - return space -end - --- start tube on space -function tube.new(space, on_task_change) - validate_space(space) - - on_task_change = on_task_change or (function() end) - local self = setmetatable({ - space = space, - on_task_change = on_task_change, - }, { __index = method }) - return self -end - --- method.grant grants provided user to all spaces of driver. -function method.grant(self, user, opts) - box.schema.user.grant(user, 'read,write', 'space', self.space.name, opts) -end - --- normalize task: cleanup all internal fields -function method.normalize_task(self, task) - return task -end - --- put task in space -function method.put(self, data, opts) - local max - - -- Taking the maximum of the index is an implicit transactions, so it is - -- always done with 'read-confirmed' mvcc isolation level. - -- It can lead to errors when trying to make parallel 'put' calls with mvcc enabled. - -- It is hapenning because 'max' for several puts in parallel will be the same since - -- read confirmed isolation level makes visible all transactions that finished the commit. - -- To fix it we wrap it with box.begin/commit and set right isolation level. - -- Current fix does not resolve that bug in situations when we already are in transaction - -- since it will open nested transactions. - -- See https://github.com/tarantool/queue/issues/207 - -- See https://www.tarantool.io/ru/doc/latest/concepts/atomic/txn_mode_mvcc/ - - if box.cfg.memtx_use_mvcc_engine and (not box.is_in_txn()) then - box.begin({txn_isolation = 'read-committed'}) - max = self.space.index.task_id:max() - box.commit() - else - max = self.space.index.task_id:max() - end - - local id = max and max[1] + 1 or 0 - local task = self.space:insert{id, state.READY, data} - self.on_task_change(task, 'put') - return task -end - --- take task -function method.take(self) - local task - -- Taking the minimum is an implicit transactions, so it is - -- always done with 'read-confirmed' mvcc isolation level. - -- It can lead to errors when trying to make parallel 'take' calls with mvcc enabled. - -- It is hapenning because 'min' for several takes in parallel will be the same since - -- read confirmed isolation level makes visible all transactions that finished the commit. - -- To fix it we wrap it with box.begin/commit and set right isolation level. - -- Current fix does not resolve that bug in situations when we already are in transaction - -- since it will open nested transactions. - -- See https://github.com/tarantool/queue/issues/207 - -- See https://www.tarantool.io/ru/doc/latest/concepts/atomic/txn_mode_mvcc/ - if box.cfg.memtx_use_mvcc_engine and (not box.is_in_txn()) then - box.begin({txn_isolation = 'read-committed'}) - task = self.space.index.status:min{state.READY} - box.commit() - else - task = self.space.index.status:min{state.READY} - end - - if task ~= nil and task[2] == state.READY then - task = self.space:update(task[1], { { '=', 2, state.TAKEN } }) - self.on_task_change(task, 'take') - return task - end -end - --- touch task -function method.touch(self, id, ttr) - error('fifo queue does not support touch') -end - --- delete task -function method.delete(self, id) - local task = self.space:get(id) - self.space:delete(id) - if task ~= nil then - task = task:transform(2, 1, state.DONE) - self.on_task_change(task, 'delete') - end - return task -end - --- release task -function method.release(self, id, opts) - local task = self.space:update(id, {{ '=', 2, state.READY }}) - if task ~= nil then - self.on_task_change(task, 'release') - end - return task -end - --- bury task -function method.bury(self, id) - local task = self.space:update(id, {{ '=', 2, state.BURIED }}) - self.on_task_change(task, 'bury') - return task -end - --- unbury several tasks -function method.kick(self, count) - for i = 1, count do - local task = self.space.index.status:min{ state.BURIED } - if task == nil then - return i - 1 - end - if task[2] ~= state.BURIED then - return i - 1 - end - - task = self.space:update(task[1], {{ '=', 2, state.READY }}) - self.on_task_change(task, 'kick') - end - return count -end - --- peek task -function method.peek(self, id) - return self.space:get{id} -end - --- get iterator to tasks in a certain state -function method.tasks_by_state(self, task_state) - return self.space.index.status:pairs(task_state) -end - -function method.truncate(self) - self.space:truncate() -end - --- This driver has no background activity. --- Implement dummy methods for the API requirement. -function method.start() - return -end - -function method.stop() - return -end - -return tube diff --git a/queue/abstract/driver/fifottl.lua b/queue/abstract/driver/fifottl.lua deleted file mode 100644 index e44a4768..00000000 --- a/queue/abstract/driver/fifottl.lua +++ /dev/null @@ -1,427 +0,0 @@ -local log = require('log') -local fiber = require('fiber') -local state = require('queue.abstract.state') - -local util = require('queue.util') -local qc = require('queue.compat') -local num_type = qc.num_type -local str_type = qc.str_type - -local tube = {} -local method = {} - -local i_id = 1 -local i_status = 2 -local i_next_event = 3 -local i_ttl = 4 -local i_ttr = 5 -local i_pri = 6 -local i_created = 7 -local i_data = 8 - -local function is_expired(task) - local dead_event = task[i_created] + task[i_ttl] - return (dead_event <= fiber.time64()) -end - --- validate space of queue -local function validate_space(space) - -- check indexes - local indexes = {'task_id', 'status', 'watch'} - for _, index in pairs(indexes) do - if space.index[index] == nil then - error(string.format('space "%s" does not have "%s" index', - space.name, index)) - end - end -end - --- create space -function tube.create_space(space_name, opts) - opts.ttl = opts.ttl or util.MAX_TIMEOUT - opts.ttr = opts.ttr or opts.ttl - opts.pri = opts.pri or 0 - - local space_opts = {} - local if_not_exists = opts.if_not_exists or false - space_opts.temporary = opts.temporary or false - space_opts.engine = opts.engine or 'memtx' - space_opts.format = { - {name = 'task_id', type = num_type()}, - {name = 'status', type = str_type()}, - {name = 'next_event', type = num_type()}, - {name = 'ttl', type = num_type()}, - {name = 'ttr', type = num_type()}, - {name = 'pri', type = num_type()}, - {name = 'created', type = num_type()}, - {name = 'data', type = '*'} - } - - -- 1 2 3 4 5 6 7, 8 - -- task_id, status, next_event, ttl, ttr, pri, created, data - local space = box.space[space_name] - if if_not_exists and space then - -- Validate the existing space. - validate_space(box.space[space_name]) - return space - end - - space = box.schema.create_space(space_name, space_opts) - space:create_index('task_id', { - type = 'tree', - parts = {i_id, num_type()} - }) - space:create_index('status', { - type = 'tree', - parts = {i_status, str_type(), i_pri, num_type(), i_id, num_type()} - }) - space:create_index('watch', { - type = 'tree', - parts = {i_status, str_type(), i_next_event, num_type()}, - unique = false - }) - return space -end - -local delayed_state = { state.DELAYED } -local ttl_states = { state.READY, state.BURIED } -local ttr_state = { state.TAKEN } - -local function fifottl_fiber_iteration(self, processed) - local now = util.time() - local task = nil - local estimated = util.MAX_TIMEOUT - - -- delayed tasks - task = self.space.index.watch:min(delayed_state) - if task and task[i_status] == state.DELAYED then - if now >= task[i_next_event] then - task = self.space:update(task[i_id], { - { '=', i_status, state.READY }, - { '=', i_next_event, task[i_created] + task[i_ttl] } - }) - self:on_task_change(task, 'delay') - estimated = 0 - processed = processed + 1 - else - estimated = tonumber(task[i_next_event] - now) / 1000000 - end - end - - -- ttl tasks - for _, state in pairs(ttl_states) do - task = self.space.index.watch:min{ state } - if task ~= nil and task[i_status] == state then - if now >= task[i_next_event] then - task = self:delete(task[i_id]):transform(2, 1, state.DONE) - self:on_task_change(task, 'ttl') - estimated = 0 - processed = processed + 1 - else - local et = tonumber(task[i_next_event] - now) / 1000000 - estimated = et < estimated and et or estimated - end - end - end - - -- ttr tasks - task = self.space.index.watch:min(ttr_state) - if task and task[i_status] == state.TAKEN then - if now >= task[i_next_event] then - task = self.space:update(task[i_id], { - { '=', i_status, state.READY }, - { '=', i_next_event, task[i_created] + task[i_ttl] } - }) - self:on_task_change(task, 'ttr') - estimated = 0 - processed = processed + 1 - else - local et = tonumber(task[i_next_event] - now) / 1000000 - estimated = et < estimated and et or estimated - end - end - - if estimated > 0 or processed > 1000 then - -- free refcounter - estimated = estimated > 0 and estimated or 0 - processed = 0 - self.cond:wait(estimated) - end - - return processed -end - --- watch fiber -local function fifottl_fiber(self) - fiber.name('fifottl') - log.info("Started queue fifottl fiber") - local processed = 0 - - while true do - if box.info.ro == false then - local stat, err = pcall(fifottl_fiber_iteration, self, processed) - - if not stat and not (err.code == box.error.READONLY) then - log.error("error catched: %s", tostring(err)) - log.error("exiting fiber '%s'", fiber.name()) - return 1 - elseif stat then - processed = err - end - else - -- When switching the master to the replica, the fiber will be stopped. - if self.sync_chan:get(0.1) ~= nil then - log.info("Queue fifottl fiber was stopped") - break - end - end - end -end - --- start tube on space -function tube.new(space, on_task_change, opts) - validate_space(space) - - on_task_change = on_task_change or (function() end) - local self = setmetatable({ - space = space, - on_task_change = function(self, task, stats_data) - -- wakeup fiber - if task ~= nil and self.fiber ~= nil then - self.cond:signal(self.fiber:id()) - end - on_task_change(task, stats_data) - end, - opts = opts, - }, { __index = method }) - - self.cond = qc.waiter() - self.fiber = fiber.create(fifottl_fiber, self) - self.sync_chan = fiber.channel() - - return self -end - --- method.grant grants provided user to all spaces of driver. -function method.grant(self, user, opts) - box.schema.user.grant(user, 'read,write', 'space', self.space.name, opts) -end - --- cleanup internal fields in task -function method.normalize_task(self, task) - return task and task:transform(3, 5) -end - --- put task in space -function method.put(self, data, opts) - local max - - -- Taking the maximum of the index is an implicit transactions, so it is - -- always done with 'read-confirmed' mvcc isolation level. - -- It can lead to errors when trying to make parallel 'put' calls with mvcc enabled. - -- It is hapenning because 'max' for several puts in parallel will be the same since - -- read confirmed isolation level makes visible all transactions that finished the commit. - -- To fix it we wrap it with box.begin/commit and set right isolation level. - -- Current fix does not resolve that bug in situations when we already are in transaction - -- since it will open nested transactions. - -- See https://github.com/tarantool/queue/issues/207 - -- See https://www.tarantool.io/ru/doc/latest/concepts/atomic/txn_mode_mvcc/ - - if box.cfg.memtx_use_mvcc_engine and (not box.is_in_txn()) then - box.begin({txn_isolation = 'read-committed'}) - max = self.space.index.task_id:max() - box.commit() - else - max = self.space.index.task_id:max() - end - - local id = max and max[i_id] + 1 or 0 - - local status - local ttl = opts.ttl or self.opts.ttl - local ttr = opts.ttr or self.opts.ttr - local pri = opts.pri or self.opts.pri or 0 - - local next_event - - if opts.delay ~= nil and opts.delay > 0 then - status = state.DELAYED - ttl = ttl + opts.delay - next_event = util.event_time(opts.delay) - else - status = state.READY - next_event = util.event_time(ttl) - end - - local task = self.space:insert{ - id, - status, - next_event, - util.time(ttl), - util.time(ttr), - pri, - util.time(), - data - } - self:on_task_change(task, 'put') - return task -end - --- touch task -function method.touch(self, id, delta) - local ops = { - {'+', i_next_event, delta}, - {'+', i_ttl, delta}, - {'+', i_ttr, delta} - } - if delta == util.MAX_TIMEOUT then - ops = { - {'=', i_next_event, delta}, - {'=', i_ttl, delta}, - {'=', i_ttr, delta} - } - end - local task = self.space:update(id, ops) - - self:on_task_change(task, 'touch') - return task -end - --- take task -function method.take(self) - local task = nil - if box.cfg.memtx_use_mvcc_engine and (not box.is_in_txn()) then - box.begin({txn_isolation = 'read-committed'}) - for _, t in self.space.index.status:pairs({state.READY}) do - if not is_expired(t) then - task = t - break - end - end - box.commit() - else - for _, t in self.space.index.status:pairs({state.READY}) do - if not is_expired(t) then - task = t - break - end - end - end - - if task == nil then - return - end - - local next_event = util.time() + task[i_ttr] - local dead_event = task[i_created] + task[i_ttl] - if next_event > dead_event then - next_event = dead_event - end - - task = self.space:update(task[i_id], { - { '=', i_status, state.TAKEN }, - { '=', i_next_event, next_event } - }) - self:on_task_change(task, 'take') - return task -end - --- delete task -function method.delete(self, id) - local task = self.space:get(id) - self.space:delete(id) - if task ~= nil then - task = task:transform(2, 1, state.DONE) - self:on_task_change(task, 'delete') - end - return task -end - --- release task -function method.release(self, id, opts) - local task = self.space:get{id} - if task == nil then - return - end - if opts.delay ~= nil and opts.delay > 0 then - task = self.space:update(id, { - { '=', i_status, state.DELAYED }, - { '=', i_next_event, util.event_time(opts.delay) }, - { '+', i_ttl, util.time(opts.delay) } - }) - else - task = self.space:update(id, { - { '=', i_status, state.READY }, - { '=', i_next_event, task[i_created] + task[i_ttl] } - }) - end - self:on_task_change(task, 'release') - return task -end - --- bury task -function method.bury(self, id) - -- The `i_next_event` should be updated because if the task has been - -- "buried" after it was "taken" (and the task has "ttr") when the time in - -- `i_next_event` will be interpreted as "ttl" in `fifottl_fiber_iteration` - -- and the task will be deleted. - local task = self.space:get{id} - if task == nil then - return - end - task = self.space:update(id, { - { '=', i_status, state.BURIED }, - { '=', i_next_event, task[i_created] + task[i_ttl] } - }) - self:on_task_change(task, 'bury') - return task -end - --- unbury several tasks -function method.kick(self, count) - for i = 1, count do - local task = self.space.index.status:min{ state.BURIED } - if task == nil then - return i - 1 - end - if task[i_status] ~= state.BURIED then - return i - 1 - end - - task = self.space:update(task[i_id], {{ '=', i_status, state.READY }}) - self:on_task_change(task, 'kick') - end - return count -end - --- peek task -function method.peek(self, id) - return self.space:get{id} -end - --- get iterator to tasks in a certain state -function method.tasks_by_state(self, task_state) - return self.space.index.status:pairs(task_state) -end - -function method.truncate(self) - self.space:truncate() -end - -function method.start(self) - if self.fiber then - return - end - self.fiber = fiber.create(fifottl_fiber, self) -end - -function method.stop(self) - if not self.fiber then - return - end - self.cond:signal(self.fiber:id()) - self.sync_chan:put(true) - self.fiber = nil -end - -return tube diff --git a/queue/abstract/driver/limfifottl.lua b/queue/abstract/driver/limfifottl.lua deleted file mode 100644 index 123a9f86..00000000 --- a/queue/abstract/driver/limfifottl.lua +++ /dev/null @@ -1,48 +0,0 @@ -local fiber = require('fiber') -local fifottl = require('queue.abstract.driver.fifottl') - -local tube = {} - -tube.create_space = function(space_name, opts) - if opts.engine == 'vinyl' then - error('limfifottl queue does not support vinyl engine') - end - return fifottl.create_space(space_name, opts) -end - --- start tube on space -function tube.new(space, on_task_change, opts) - local state = { - capacity = opts.capacity or 0, - parent = fifottl.new(space, on_task_change, opts) - } - - -- put task in space - local put = function (self, data, opts) - local timeout = opts.timeout or 0 - local started = tonumber(fiber.time()) - - while true do - local tube_size = self.space:len() - if tube_size < state.capacity or state.capacity == 0 then - return state.parent.put(self, data, opts) - else - if tonumber(fiber.time()) - started > timeout then - return nil - end - fiber.sleep(.01) - end - end - end - - local len = function (self) - return self.space:len() - end - - return setmetatable({ - put = put, - len = len - }, {__index = state.parent}) -end - -return tube diff --git a/queue/abstract/driver/utube.lua b/queue/abstract/driver/utube.lua deleted file mode 100644 index 0a7faf94..00000000 --- a/queue/abstract/driver/utube.lua +++ /dev/null @@ -1,469 +0,0 @@ -local state = require('queue.abstract.state') -local num_type = require('queue.compat').num_type -local str_type = require('queue.compat').str_type - -local tube = {} -local method = {} - -tube.STORAGE_MODE_DEFAULT = "default" -tube.STORAGE_MODE_READY_BUFFER = "ready_buffer" - -local i_status = 2 - --- validate space of queue -local function validate_space(space) - -- check indexes - local indexes = {'task_id', 'status', 'utube'} - for _, index in pairs(indexes) do - if space.index[index] == nil then - error(string.format('space "%s" does not have "%s" index', - space.name, index)) - end - end -end - --- validate ready buffer space of queue -local function validate_space_ready_buffer(space) - -- check indexes - local indexes = {'task_id', 'utube'} - for _, index in pairs(indexes) do - if space.index[index] == nil then - error(string.format('space "%s" does not have "%s" index', - space.name, index)) - end - end -end - --- create space -function tube.create_space(space_name, opts) - local space_opts = {} - local if_not_exists = opts.if_not_exists or false - space_opts.temporary = opts.temporary or false - space_opts.engine = opts.engine or 'memtx' - space_opts.format = { - {name = 'task_id', type = num_type()}, - {name = 'status', type = str_type()}, - {name = 'utube', type = str_type()}, - {name = 'data', type = '*'} - } - - -- id, status, utube, data - local space = box.space[space_name] - if if_not_exists and space then - -- Validate the existing space. - validate_space(box.space[space_name]) - return space - end - - space = box.schema.create_space(space_name, space_opts) - space:create_index('task_id', { - type = 'tree', - parts = {1, num_type()} - }) - space:create_index('status', { - type = 'tree', - parts = {2, str_type(), 1, num_type()} - }) - space:create_index('utube', { - type = 'tree', - parts = {2, str_type(), 3, str_type(), 1, num_type()} - }) - return space -end - --- start tube on space -function tube.new(space, on_task_change, opts) - validate_space(space) - - local space_opts = {} - space_opts.temporary = opts.temporary or false - space_opts.engine = opts.engine or 'memtx' - space_opts.format = { - {name = 'task_id', type = num_type()}, - {name = 'utube', type = str_type()} - } - - local space_ready_buffer_name = space.name .. "_ready_buffer" - local space_ready_buffer = box.space[space_ready_buffer_name] - -- Feature implemented only for memtx engine for now. - -- https://github.com/tarantool/queue/issues/230. - if opts.storage_mode == tube.STORAGE_MODE_READY_BUFFER and opts.engine == 'vinyl' then - error(string.format('"%s" storage mode cannot be used with vinyl engine', - tube.STORAGE_MODE_READY_BUFFER)) - end - - local ready_space_mode = (opts.storage_mode == tube.STORAGE_MODE_READY_BUFFER) - if ready_space_mode then - if space_ready_buffer == nil then - -- Create a space for first ready tasks from each utube. - space_ready_buffer = box.schema.create_space(space_ready_buffer_name, space_opts) - space_ready_buffer:create_index('task_id', { - type = 'tree', - parts = {1, num_type()}, - unique = true, - }) - space_ready_buffer:create_index('utube', { - type = 'tree', - parts = {2, str_type()}, - unique = true, - }) - else - validate_space_ready_buffer(space_ready_buffer) - if space:len() == 0 then - space_ready_buffer:truncate() - end - end - end - - on_task_change = on_task_change or (function() end) - local self = setmetatable({ - space = space, - space_ready_buffer = space_ready_buffer, - on_task_change = on_task_change, - ready_space_mode = ready_space_mode, - opts = opts, - }, { __index = method }) - return self -end - --- method.grant grants provided user to all spaces of driver. -function method.grant(self, user, opts) - box.schema.user.grant(user, 'read,write', 'space', self.space.name, opts) - if self.space_ready_buffer ~= nil then - box.schema.user.grant(user, 'read,write', 'space', self.space_ready_buffer.name, opts) - end -end - --- normalize task: cleanup all internal fields -function method.normalize_task(self, task) - return task and task:transform(3, 1) -end - --- Find the first ready task for given 'utube'. --- Utube is also checked for the absence of 'TAKEN' tasks. -local function put_next_ready(self, utube) - local taken = self.space.index.utube:min{state.TAKEN, utube} - if taken == nil or taken[2] ~= state.TAKEN then - local next_task = self.space.index.utube:min{state.READY, utube} - if next_task == nil or next_task[2] ~= state.READY then - return - end - -- Ignoring ER_TUPLE_FOUND error, if a tuple with the same task_id - -- or utube name is already in the space. - -- Note that both task_id and utube indexes are unique, so there will be - -- no duplicates: each task_id can occur in the space not more than once, - -- there can be no more than one task from each utube in a space. - pcall(self.space_ready_buffer.insert, self.space_ready_buffer, {next_task[1], utube}) - end -end - --- Put this task into ready_buffer. --- Utube is also checked for the absence of 'TAKEN' tasks. -local function put_ready(self, id, utube) - local taken = self.space.index.utube:min{state.TAKEN, utube} - if taken == nil or taken[2] ~= state.TAKEN then - -- Ignoring ER_TUPLE_FOUND error, if a tuple with the same task_id - -- or utube name is already in the space. - -- Note that both task_id and utube indexes are unique, so there will be - -- no duplicates: each task_id can occur in the space not more than once, - -- there can be no more than one task from each utube in a space. - pcall(self.space_ready_buffer.insert, self.space_ready_buffer, {id, utube}) - end -end - -local function commit() - box.commit() -end - -local function empty() -end - --- Start transaction with the correct options, if the transaction is not already running. -local function begin_if_not_in_txn() - local transaction_opts = {} - if box.cfg.memtx_use_mvcc_engine then - transaction_opts = {txn_isolation = 'read-committed'} - end - - if not box.is_in_txn() then - box.begin(transaction_opts) - return commit - else - return empty - end -end - --- put task in space -function method.put(self, data, opts) - -- Taking the minimum is an implicit transactions, so it is - -- always done with 'read-confirmed' mvcc isolation level. - -- It can lead to errors when trying to make parallel 'take' calls with mvcc enabled. - -- It is hapenning because 'min' for several takes in parallel will be the same since - -- read confirmed isolation level makes visible all transactions that finished the commit. - -- To fix it we wrap it with box.begin/commit and set right isolation level. - -- Current fix does not resolve that bug in situations when we already are in transaction - -- since it will open nested transactions. - -- See https://github.com/tarantool/queue/issues/207 - -- See https://www.tarantool.io/ru/doc/latest/concepts/atomic/txn_mode_mvcc/ - local commit_func = begin_if_not_in_txn() - - local max = self.space.index.task_id:max() - - local id = max and max[1] + 1 or 0 - local task = self.space:insert{id, state.READY, tostring(opts.utube), data} - if self.ready_space_mode then - put_ready(self, task[1], task[3]) - end - - commit_func() - - self.on_task_change(task, 'put') - return task -end - --- Take the first task form the ready_buffer. -local function take_ready(self) - while true do - local commit_func = begin_if_not_in_txn() - - local task_ready = self.space_ready_buffer.index.task_id:min() - if task_ready == nil then - commit_func() - return nil - end - - local id = task_ready[1] - local task = self.space:get(id) - local take_complete = false - - if task[2] == state.READY then - local taken = self.space.index.utube:min{state.TAKEN, task[3]} - - if taken == nil or taken[2] ~= state.TAKEN then - task = self.space:update(id, { { '=', 2, state.TAKEN } }) - self.space_ready_buffer:delete(id) - take_complete = true - end - end - - commit_func() - - if take_complete then - self.on_task_change(task, 'take') - return task - end - end -end - -local function take(self) - for s, task in self.space.index.status:pairs(state.READY, - { iterator = 'GE' }) do - if task[2] ~= state.READY then - break - end - -- Taking the minimum is an implicit transactions, so it is - -- always done with 'read-confirmed' mvcc isolation level. - -- It can lead to errors when trying to make parallel 'take' calls with mvcc enabled. - -- It is hapenning because 'min' for several takes in parallel will be the same since - -- read confirmed isolation level makes visible all transactions that finished the commit. - -- To fix it we wrap it with box.begin/commit and set right isolation level. - -- Current fix does not resolve that bug in situations when we already are in transaction - -- since it will open nested transactions. - -- See https://github.com/tarantool/queue/issues/207 - -- See https://www.tarantool.io/ru/doc/latest/concepts/atomic/txn_mode_mvcc/ - local commit_func = begin_if_not_in_txn() - local taken = self.space.index.utube:min{state.TAKEN, task[3]} - local take_complete = false - - if taken == nil or taken[2] ~= state.TAKEN then - task = self.space:update(task[1], { { '=', 2, state.TAKEN } }) - take_complete = true - end - - commit_func() - if take_complete then - self.on_task_change(task, 'take') - return task - end - end -end - --- take task -function method.take(self) - if self.ready_space_mode then - return take_ready(self) - end - return take(self) -end - --- touch task -function method.touch(self, id, ttr) - error('utube queue does not support touch') -end - --- Delete task from the ready_buffer and find next ready task from the same 'utube' to replace it. -local function delete_ready(self, id, utube) - self.space_ready_buffer:delete(id) - put_next_ready(self, utube) -end - --- delete task -function method.delete(self, id) - local commit_func = begin_if_not_in_txn() - - local task = self.space:get(id) - self.space:delete(id) - if task ~= nil then - if self.ready_space_mode then - if task[2] == state.TAKEN then - put_next_ready(self, task[3]) - elseif task[2] == state.READY then - delete_ready(self, id, task[3]) - end - end - - task = task:transform(2, 1, state.DONE) - - local neighbour = self.space.index.utube:min{state.READY, task[3]} - - commit_func() - - self.on_task_change(task, 'delete') - if neighbour then - self.on_task_change(neighbour) - end - return task - end - - commit_func() - return task -end - --- release task -function method.release(self, id, opts) - local commit_func = begin_if_not_in_txn() - - local task = self.space:update(id, {{ '=', 2, state.READY }}) - if task ~= nil then - if self.ready_space_mode then - local inserted, err = - pcall(self.space_ready_buffer.insert, self.space_ready_buffer, {id, task[3]}) - if not inserted then - require('log').warn( - 'queue: [tube "utube"] insert after release error: %s', err) - delete_ready(self, task[1], task[3]) - end - end - - commit_func() - - self.on_task_change(task, 'release') - return task - end - - commit_func() - return task -end - --- bury task -function method.bury(self, id) - local commit_func = begin_if_not_in_txn() - - local current_task = self.space:get{id} - local task = self.space:update(id, {{ '=', 2, state.BURIED }}) - if task ~= nil then - if self.ready_space_mode then - local status = current_task[2] - local ready_task = self.space_ready_buffer:get{task[1]} - if ready_task ~= nil then - delete_ready(self, id, task[3]) - elseif status == state.TAKEN then - put_next_ready(self, task[3]) - end - end - - local neighbour = self.space.index.utube:min{state.READY, task[3]} - - commit_func() - - self.on_task_change(task, 'bury') - if neighbour and neighbour[i_status] == state.READY then - self.on_task_change(neighbour) - end - else - commit_func() - - self.on_task_change(task, 'bury') - end - return task -end - --- unbury several tasks -function method.kick(self, count) - for i = 1, count do - local commit_func = begin_if_not_in_txn() - - local task = self.space.index.status:min{ state.BURIED } - if task == nil then - return i - 1 - end - if task[2] ~= state.BURIED then - return i - 1 - end - - task = self.space:update(task[1], {{ '=', 2, state.READY }}) - if self.ready_space_mode then - local prev_task = self.space_ready_buffer.index.utube:get{task[3]} - if prev_task ~= nil then - if prev_task[1] > task[1] then - self.space_ready_buffer:delete(prev_task[1]) - self.space_ready_buffer:insert({task[1], task[2]}) - end - else - put_ready(self, task[3]) - end - end - - commit_func() - - self.on_task_change(task, 'kick') - end - return count -end - --- peek task -function method.peek(self, id) - return self.space:get{id} -end - --- get iterator to tasks in a certain state -function method.tasks_by_state(self, task_state) - return self.space.index.status:pairs(task_state) -end - -function method.truncate(self) - self.space:truncate() - if self.ready_space_mode then - self.space_ready_buffer:truncate() - end -end - --- This driver has no background activity. --- Implement dummy methods for the API requirement. -function method.start() - return -end - -function method.stop() - return -end - -function method.drop(self) - self:stop() - box.space[self.space.name]:drop() - if self.ready_space_mode then - box.space[self.space_ready_buffer.name]:drop() - end -end - -return tube diff --git a/queue/abstract/driver/utubettl.lua b/queue/abstract/driver/utubettl.lua deleted file mode 100644 index 7b726ba5..00000000 --- a/queue/abstract/driver/utubettl.lua +++ /dev/null @@ -1,764 +0,0 @@ -local log = require('log') -local fiber = require('fiber') - -local state = require('queue.abstract.state') - -local util = require('queue.util') -local qc = require('queue.compat') -local num_type = qc.num_type -local str_type = qc.str_type - -local tube = {} -local method = {} - -tube.STORAGE_MODE_DEFAULT = "default" -tube.STORAGE_MODE_READY_BUFFER = "ready_buffer" - -local i_id = 1 -local i_status = 2 -local i_next_event = 3 -local i_ttl = 4 -local i_ttr = 5 -local i_pri = 6 -local i_created = 7 -local i_utube = 8 -local i_data = 9 - -local function is_expired(task) - local dead_event = task[i_created] + task[i_ttl] - return (dead_event <= fiber.time64()) -end - --- validate space of queue -local function validate_space(space) - -- check indexes - local indexes = {'task_id', 'status', 'utube', 'watch', 'utube_pri'} - for _, index in pairs(indexes) do - if space.index[index] == nil then - error(string.format('space "%s" does not have "%s" index', - space.name, index)) - end - end -end - --- validate ready buffer space of queue -local function validate_space_ready_buffer(space) - -- check indexes - local indexes = {'task_id', 'utube', 'pri'} - for _, index in pairs(indexes) do - if space.index[index] == nil then - error(string.format('space "%s" does not have "%s" index', - space.name, index)) - end - end -end - --- create space -function tube.create_space(space_name, opts) - opts.ttl = opts.ttl or util.MAX_TIMEOUT - opts.ttr = opts.ttr or opts.ttl - opts.pri = opts.pri or 0 - - local space_opts = {} - local if_not_exists = opts.if_not_exists or false - space_opts.temporary = opts.temporary or false - space_opts.engine = opts.engine or 'memtx' - space_opts.format = { - {name = 'task_id', type = num_type()}, - {name = 'status', type = str_type()}, - {name = 'next_event', type = num_type()}, - {name = 'ttl', type = num_type()}, - {name = 'ttr', type = num_type()}, - {name = 'pri', type = num_type()}, - {name = 'created', type = num_type()}, - {name = 'utube', type = str_type()}, - {name = 'data', type = '*'} - } - - -- 1 2 3 4 5 6 7, 8 9 - -- task_id, status, next_event, ttl, ttr, pri, created, utube, data - local space = box.space[space_name] - if if_not_exists and space then - -- Validate the existing space. - validate_space(box.space[space_name]) - return space - end - - space = box.schema.create_space(space_name, space_opts) - space:create_index('task_id', { - type = 'tree', - parts = {i_id, num_type()} - }) - space:create_index('status', { - type = 'tree', - parts = {i_status, str_type(), i_pri, num_type(), i_id, num_type()} - }) - space:create_index('watch', { - type = 'tree', - parts = {i_status, str_type(), i_next_event, num_type()}, - unique = false - }) - space:create_index('utube', { - type = 'tree', - parts = {i_status, str_type(), i_utube, str_type(), i_id, num_type()} - }) - space:create_index('utube_pri', { - type = 'tree', - parts = {i_status, str_type(), i_utube, str_type(), i_pri, num_type(), i_id, num_type()} - }) - return space -end - -local delayed_state = { state.DELAYED } -local ttl_states = { state.READY, state.BURIED } -local ttr_state = { state.TAKEN } - --- Find the first ready task for given 'utube'. --- Utube is also checked for the absence of 'TAKEN' tasks. -local function put_next_ready(self, utube) - local taken = self.space.index.utube:min{state.TAKEN, utube} - if taken == nil or taken[i_status] ~= state.TAKEN then - local next_task = self.space.index.utube_pri:min{state.READY, utube} - if next_task == nil or next_task[i_status] ~= state.READY then - return - end - -- Ignoring ER_TUPLE_FOUND error, if a tuple with the same task_id - -- or utube name is already in the space. - -- Note that both task_id and utube indexes are unique, so there will be - -- no duplicates: each task_id can occur in the space not more than once, - -- there can be no more than one task from each utube in a space. - pcall(self.space_ready_buffer.insert, self.space_ready_buffer, {next_task[i_id], utube, next_task[i_pri]}) - end -end - --- Check if given task has lowest priority in the ready_buffer for the given 'utube'. --- If so, current task for the given 'utube' is replaced in the ready_buffer. --- Utube is also checked for the absence of 'TAKEN' tasks. -local function put_ready(self, id, utube, pri) - local taken = self.space.index.utube:min{state.TAKEN, utube} - if taken == nil or taken[i_status] ~= state.TAKEN then - local cur_task = self.space_ready_buffer.index.utube:get{utube} - - if cur_task ~= nil then - local cur_id = cur_task[1] - local cur_pri = cur_task[3] - if cur_pri < pri or (cur_pri == pri and cur_id < id) then - return - end - self.space_ready_buffer:delete(cur_id) - end - - -- Ignoring ER_TUPLE_FOUND error, if a tuple with the same task_id - -- or utube name is already in the space. - -- Note that both task_id and utube indexes are unique, so there will be - -- no duplicates: each task_id can occur in the space not more than once, - -- there can be no more than one task from each utube in a space. - pcall(self.space_ready_buffer.insert, self.space_ready_buffer, {id, utube, pri}) - end -end - --- Delete task from the ready_buffer and find next ready task from the same 'utube' to replace it. -local function delete_ready(self, id, utube) - self.space_ready_buffer:delete(id) - put_next_ready(self, utube) -end - --- Try to update the current task in ready_buffer for the given 'utube'. -local function update_ready(self, id, utube, pri) - local prev_task = self.space_ready_buffer.index.utube:get{utube} - if prev_task ~= nil then - if prev_task[3] > pri or (prev_task[3] == pri and prev_task[1] > id) then - self.space_ready_buffer:delete(prev_task[1]) - self.space_ready_buffer:insert({id, utube, pri}) - end - else - put_ready(self, id, utube, pri) - end -end - -local function commit() - box.commit() -end - -local function empty() -end - --- Start transaction with the correct options, if the transaction is not already running --- and current engine is not 'vinyl'. -local function begin_if_not_in_txn(self) - local transaction_opts = {} - if box.cfg.memtx_use_mvcc_engine then - transaction_opts = {txn_isolation = 'read-committed'} - end - - -- Implemented only for memtx engine for now. - -- https://github.com/tarantool/queue/issues/230. - if not box.is_in_txn() and self.opts.engine ~= 'vinyl' then - box.begin(transaction_opts) - return commit - else - return empty - end -end - -local function utubettl_fiber_iteration(self, processed) - local now = util.time() - local task = nil - local estimated = util.MAX_TIMEOUT - - local commit_func = begin_if_not_in_txn(self) - local commited = false - - -- delayed tasks - task = self.space.index.watch:min(delayed_state) - if task and task[i_status] == state.DELAYED then - if now >= task[i_next_event] then - task = self.space:update(task[i_id], { - { '=', i_status, state.READY }, - { '=', i_next_event, task[i_created] + task[i_ttl] } - }) - - if self.ready_space_mode then - update_ready(self, task[i_id], task[i_utube], task[i_pri]) - end - commit_func() - commited = true - - self:on_task_change(task, 'delayed') - estimated = 0 - processed = processed + 1 - else - estimated = tonumber(task[i_next_event] - now) / 1000000 - end - end - if not commited then - commit_func() - end - - -- ttl tasks - for _, state in pairs(ttl_states) do - task = self.space.index.watch:min{ state } - if task ~= nil and task[i_status] == state then - if now >= task[i_next_event] then - task = self:delete(task[i_id]):transform(2, 1, state.DONE) - self:on_task_change(task, 'ttl') - estimated = 0 - processed = processed + 1 - else - local et = tonumber(task[i_next_event] - now) / 1000000 - estimated = et < estimated and et or estimated - end - end - end - - commit_func = begin_if_not_in_txn(self) - commited = false - -- ttr tasks - task = self.space.index.watch:min(ttr_state) - if task and task[i_status] == state.TAKEN then - if now >= task[i_next_event] then - task = self.space:update(task[i_id], { - { '=', i_status, state.READY }, - { '=', i_next_event, task[i_created] + task[i_ttl] } - }) - - if self.ready_space_mode then - put_ready(self, task[i_id], task[i_utube], task[i_pri]) - end - commit_func() - commited = true - - self:on_task_change(task, 'ttr') - estimated = 0 - processed = processed + 1 - else - local et = tonumber(task[i_next_event] - now) / 1000000 - estimated = et < estimated and et or estimated - end - end - if not commited then - commit_func() - end - - if estimated > 0 or processed > 1000 then - -- free refcounter - estimated = processed > 1000 and 0 or estimated - estimated = estimated > 0 and estimated or 0 - processed = 0 - self.cond:wait(estimated) - end - - return processed -end - --- watch fiber -local function utubettl_fiber(self) - fiber.name('utubettl') - log.info("Started queue utubettl fiber") - local processed = 0 - - while true do - if box.info.ro == false then - local stat, err = pcall(utubettl_fiber_iteration, self, processed) - - if not stat and not (err.code == box.error.READONLY) then - log.error("error catched: %s", tostring(err)) - log.error("exiting fiber '%s'", fiber.name()) - return 1 - elseif stat then - processed = err - end - else - -- When switching the master to the replica, the fiber will be stopped. - if self.sync_chan:get(0.1) ~= nil then - log.info("Queue utubettl fiber was stopped") - break - end - end - end -end - --- start tube on space -function tube.new(space, on_task_change, opts) - validate_space(space) - - local space_ready_buffer_name = space.name .. "_ready_buffer" - local space_ready_buffer = box.space[space_ready_buffer_name] - -- Feature implemented only for memtx engine for now. - -- https://github.com/tarantool/queue/issues/230. - if opts.storage_mode == tube.STORAGE_MODE_READY_BUFFER and opts.engine == 'vinyl' then - error(string.format('"%s" storage mode cannot be used with vinyl engine', - tube.STORAGE_MODE_READY_BUFFER)) - end - - local ready_space_mode = (opts.storage_mode == tube.STORAGE_MODE_READY_BUFFER or false) - if ready_space_mode then - if space_ready_buffer == nil then - local space_opts = {} - space_opts.temporary = opts.temporary or false - space_opts.engine = opts.engine or 'memtx' - space_opts.format = { - {name = 'task_id', type = num_type()}, - {name = 'utube', type = str_type()}, - {name = 'pri', type = num_type()}, - } - - -- Create a space for first ready tasks from each utube. - space_ready_buffer = box.schema.create_space(space_ready_buffer_name, space_opts) - space_ready_buffer:create_index('task_id', { - type = 'tree', - parts = {1, num_type()}, - unique = true, - }) - space_ready_buffer:create_index('utube', { - type = 'tree', - parts = {2, str_type()}, - unique = true, - }) - space_ready_buffer:create_index('pri', { - type = 'tree', - parts = {3, num_type(), 1, num_type()}, - }) - else - validate_space_ready_buffer(space_ready_buffer) - if space:len() == 0 then - space_ready_buffer:truncate() - end - end - end - - on_task_change = on_task_change or (function() end) - local self = setmetatable({ - space = space, - space_ready_buffer = space_ready_buffer, - on_task_change = function(self, task, stat_data) - -- wakeup fiber - if task ~= nil and self.fiber ~= nil then - self.cond:signal(self.fiber:id()) - end - on_task_change(task, stat_data) - end, - opts = opts, - ready_space_mode = ready_space_mode, - }, { __index = method }) - - self.cond = qc.waiter() - self.fiber = fiber.create(utubettl_fiber, self) - self.sync_chan = fiber.channel(1) - - return self -end - --- method.grant grants provided user to all spaces of driver. -function method.grant(self, user, opts) - box.schema.user.grant(user, 'read,write', 'space', self.space.name, opts) - if self.space_ready_buffer ~= nil then - box.schema.user.grant(user, 'read,write', 'space', self.space_ready_buffer.name, opts) - end -end - --- cleanup internal fields in task -function method.normalize_task(self, task) - return task and task:transform(i_next_event, i_data - i_next_event) -end - --- put task in space -function method.put(self, data, opts) - -- Taking the minimum is an implicit transactions, so it is - -- always done with 'read-confirmed' mvcc isolation level. - -- It can lead to errors when trying to make parallel 'take' calls with mvcc enabled. - -- It is hapenning because 'min' for several takes in parallel will be the same since - -- read confirmed isolation level makes visible all transactions that finished the commit. - -- To fix it we wrap it with box.begin/commit and set right isolation level. - -- Current fix does not resolve that bug in situations when we already are in transaction - -- since it will open nested transactions. - -- See https://github.com/tarantool/queue/issues/207 - -- See https://www.tarantool.io/ru/doc/latest/concepts/atomic/txn_mode_mvcc/ - local commit_func = begin_if_not_in_txn(self) - - local max = self.space.index.task_id:max() - local id = max and max[i_id] + 1 or 0 - - local status - local ttl = opts.ttl or self.opts.ttl - local ttr = opts.ttr or self.opts.ttr - local pri = opts.pri or self.opts.pri or 0 - - local next_event - - if opts.delay ~= nil and opts.delay > 0 then - status = state.DELAYED - ttl = ttl + opts.delay - next_event = util.event_time(opts.delay) - else - status = state.READY - next_event = util.event_time(ttl) - end - - local task = self.space:insert{ - id, - status, - next_event, - util.time(ttl), - util.time(ttr), - pri, - util.time(), - tostring(opts.utube), - data - } - if self.ready_space_mode and status == state.READY then - put_ready(self, task[i_id], task[i_utube], task[i_pri]) - end - - commit_func() - - self:on_task_change(task, 'put') - return task -end - -local TIMEOUT_INFINITY_TIME = util.time(util.MAX_TIMEOUT) - --- touch task -function method.touch(self, id, delta) - local ops = { - {'+', i_next_event, delta}, - {'+', i_ttl, delta}, - {'+', i_ttr, delta} - } - if delta == util.MAX_TIMEOUT then - ops = { - {'=', i_next_event, delta}, - {'=', i_ttl, delta}, - {'=', i_ttr, delta} - } - end - local task = self.space:update(id, ops) - - self:on_task_change(task, 'touch') - return task -end - --- Take the first task form the ready_buffer. -local function take_ready(self) - while true do - local commit_func = begin_if_not_in_txn(self) - - local task_ready = self.space_ready_buffer.index.pri:min() - if task_ready == nil then - commit_func() - return nil - end - - local id = task_ready[1] - local task = self.space:get(id) - local take_complete = false - local take_ttl = false - - if task[i_status] == state.READY then - if not is_expired(task) then - local next_event = util.time() + task[i_ttr] - - local taken = self.space.index.utube:min{state.TAKEN, task[i_utube]} - - if taken == nil or taken[i_status] ~= state.TAKEN then - task = self.space:update(task[1], { - { '=', i_status, state.TAKEN }, - { '=', i_next_event, next_event } - }) - - self.space_ready_buffer:delete(task[i_id]) - take_complete = true - end - else - task = self:delete(task[i_id]):transform(2, 1, state.DONE) - take_ttl = true - end - end - - commit_func() - - if take_complete then - self:on_task_change(task, 'take') - return task - elseif take_ttl then - self:on_task_change(task, 'ttl') - end - end -end - -local function take(self) - for s, t in self.space.index.status:pairs(state.READY, {iterator = 'GE'}) do - if t[2] ~= state.READY then - break - elseif not is_expired(t) then - local next_event = util.time() + t[i_ttr] - -- Taking the minimum is an implicit transactions, so it is - -- always done with 'read-confirmed' mvcc isolation level. - -- It can lead to errors when trying to make parallel 'take' calls with mvcc enabled. - -- It is hapenning because 'min' for several takes in parallel will be the same since - -- read confirmed isolation level makes visible all transactions that finished the commit. - -- To fix it we wrap it with box.begin/commit and set right isolation level. - -- Current fix does not resolve that bug in situations when we already are in transaction - -- since it will open nested transactions. - -- See https://github.com/tarantool/queue/issues/207 - -- See https://www.tarantool.io/ru/doc/latest/concepts/atomic/txn_mode_mvcc/ - local commit_func = begin_if_not_in_txn(self) - local taken = self.space.index.utube:min{state.TAKEN, t[i_utube]} - local take_complete = false - - if taken == nil or taken[i_status] ~= state.TAKEN then - t = self.space:update(t[1], { - { '=', i_status, state.TAKEN }, - { '=', i_next_event, next_event } - }) - take_complete = true - end - - commit_func() - if take_complete then - self:on_task_change(t, 'take') - return t - end - end - end -end - --- take task -function method.take(self) - if self.ready_space_mode then - return take_ready(self) - end - return take(self) -end - -local function process_neighbour(self, task, operation) - self:on_task_change(task, operation) - if task ~= nil then - local neighbour = self.space.index.utube:select( - {state.READY, task[i_utube]}, - {limit = 1} - )[1] - if neighbour ~= nil then - self:on_task_change(neighbour) - end - end - return task -end - --- delete task -function method.delete(self, id) - local commit_func = begin_if_not_in_txn(self) - - local task = self.space:get(id) - if task ~= nil then - local is_taken = task[i_status] == state.TAKEN - self.space:delete(id) - - if self.ready_space_mode then - if task[i_status] == state.TAKEN then - put_next_ready(self, task[i_utube]) - elseif task[i_status] == state.READY then - delete_ready(self, id, task[i_utube]) - end - end - - task = task:transform(i_status, 1, state.DONE) - - commit_func() - - if is_taken then - return process_neighbour(self, task, 'delete') - else - self:on_task_change(task, 'delete') - return task - end - end - - commit_func() -end - --- release task -function method.release(self, id, opts) - local commit_func = begin_if_not_in_txn(self) - - local task = self.space:get{id} - if task == nil then - commit_func() - return - end - if opts.delay ~= nil and opts.delay > 0 then - task = self.space:update(id, { - { '=', i_status, state.DELAYED }, - { '=', i_next_event, util.event_time(opts.delay) }, - { '+', i_ttl, util.time(opts.delay) } - }) - if task ~= nil then - if self.ready_space_mode then - put_next_ready(self, task[i_utube]) - end - - commit_func() - - return process_neighbour(self, task, 'release') - end - else - task = self.space:update(id, { - { '=', i_status, state.READY }, - { '=', i_next_event, util.time(task[i_created] + task[i_ttl]) } - }) - - if self.ready_space_mode and task ~= nil then - put_ready(self, task[i_id], task[i_utube], task[i_pri]) - end - end - - commit_func() - self:on_task_change(task, 'release') - return task -end - --- bury task -function method.bury(self, id) - local commit_func = begin_if_not_in_txn(self) - - -- The `i_next_event` should be updated because if the task has been - -- "buried" after it was "taken" (and the task has "ttr") when the time in - -- `i_next_event` will be interpreted as "ttl" in `utubettl_fiber_iteration` - -- and the task will be deleted. - local task = self.space:get{id} - if task == nil then - commit_func() - return - end - - local status = task[i_status] - task = self.space:update(id, { - { '=', i_status, state.BURIED }, - { '=', i_next_event, task[i_created] + task[i_ttl] } - }) - if self.ready_space_mode then - if status == state.READY then - delete_ready(self, id, task[i_utube]) - elseif status == state.TAKEN then - put_next_ready(self, task[i_utube]) - end - end - - commit_func() - - return process_neighbour( - self, task:transform(i_status, 1, state.BURIED), 'bury' - ) -end - --- unbury several tasks -function method.kick(self, count) - for i = 1, count do - local commit_func = begin_if_not_in_txn(self) - - local task = self.space.index.status:min{ state.BURIED } - if task == nil then - commit_func() - - return i - 1 - end - if task[i_status] ~= state.BURIED then - commit_func() - - return i - 1 - end - - task = self.space:update(task[i_id], {{ '=', i_status, state.READY }}) - if self.ready_space_mode then - update_ready(self, task[i_id], task[i_utube], task[i_pri]) - end - - commit_func() - - self:on_task_change(task, 'kick') - end - return count -end - --- peek task -function method.peek(self, id) - return self.space:get{id} -end - --- get iterator to tasks in a certain state -function method.tasks_by_state(self, task_state) - return self.space.index.status:pairs(task_state) -end - -function method.truncate(self) - self.space:truncate() - if self.ready_space_mode then - self.space_ready_buffer:truncate() - end -end - -function method.start(self) - if self.fiber then - return - end - self.fiber = fiber.create(utubettl_fiber, self) -end - -function method.stop(self) - if not self.fiber then - return - end - self.cond:signal(self.fiber:id()) - self.sync_chan:put(true) - self.fiber = nil -end - -function method.drop(self) - self:stop() - box.space[self.space.name]:drop() - if self.ready_space_mode then - box.space[self.space_ready_buffer.name]:drop() - end -end - -return tube diff --git a/queue/abstract/queue_session.lua b/queue/abstract/queue_session.lua deleted file mode 100644 index 9f32a8e0..00000000 --- a/queue/abstract/queue_session.lua +++ /dev/null @@ -1,358 +0,0 @@ -local log = require('log') -local fiber = require('fiber') -local uuid = require('uuid') - -local util = require('queue.util') -local qc = require('queue.compat') -local num_type = qc.num_type -local str_type = qc.str_type - --- since: --- https://github.com/tarantool/tarantool/commit/f266559b167b4643c377724eebde9651877d6cc9 -local ddl_txn_supported = qc.check_version({2, 2, 1}) - -local queue_session = {} - --- --- Replicaset mode switch. --- --- Running a queue in master-replica mode requires that --- `_queue_shared_sessions` space was not temporary. --- When the queue is running in single mode, --- the space is converted to temporary mode to increase performance. --- -local function switch_in_replicaset(replicaset_mode) - if replicaset_mode == nil then - replicaset_mode = false - end - - if not box.space._queue_shared_sessions then - return - end - - if box.space._queue_shared_sessions.temporary - and replicaset_mode == false then - return - end - - if not box.space._queue_shared_sessions.temporary - and replicaset_mode == true then - return - end - - box.schema.create_space('_queue_shared_sessions_mgr', { - temporary = not replicaset_mode, - format = { - { name = 'uuid', type = str_type() }, - { name = 'exp_time', type = num_type() }, - { name = 'active', type = 'boolean' }, - } - }) - - box.space._queue_shared_sessions_mgr:create_index('uuid', { - type = 'tree', - parts = { 1, str_type() }, - unique = true - }) - box.space._queue_shared_sessions_mgr:create_index('active', { - type = 'tree', - parts = { 3, 'boolean', 1, str_type() }, - unique = true - }) - - box.begin() -- Disable implicit yields until the transaction ends. - for _, tuple in box.space._queue_shared_sessions:pairs() do - box.space._queue_shared_sessions_mgr:insert(tuple) - end - - if ddl_txn_supported then - -- We can do it inside a transaction. - box.space._queue_shared_sessions:drop() - box.space._queue_shared_sessions_mgr:rename('_queue_shared_sessions') - end - - local status, err = pcall(box.commit) - if not status then - error(('Error migrate _queue_shared_sessions: %s'):format(tostring(err))) - end - - if not ddl_txn_supported then - -- Do it outside a transaction becase DDL does not support - -- multi-statement transactions. - box.space._queue_shared_sessions:drop() - box.space._queue_shared_sessions_mgr:rename('_queue_shared_sessions') - end -end - ---- Create everything that's needed to work with "shared" sessions. -local function identification_init() - local queue_session_ids = box.space._queue_session_ids - if queue_session_ids == nil then - queue_session_ids = box.schema.create_space('_queue_session_ids', { - temporary = true, - format = { - { name = 'connection_id', type = num_type() }, - { name = 'session_uuid', type = str_type() } - } - }) - - queue_session_ids:create_index('conn_id', { - type = 'tree', - parts = {1, num_type()}, - unique = true - }) - queue_session_ids:create_index('uuid', { - type = 'tree', - parts = {2, str_type()}, - unique = false - }) - end - - local replicaset_mode = queue_session.cfg['in_replicaset'] or false - if box.space._queue_shared_sessions == nil then - box.schema.create_space('_queue_shared_sessions', { - temporary = not replicaset_mode, - format = { - { name = 'uuid', type = str_type() }, - { name = 'exp_time', type = num_type() }, - { name = 'active', type = 'boolean' }, - } - }) - - box.space._queue_shared_sessions:create_index('uuid', { - type = 'tree', - parts = { 1, str_type() }, - unique = true - }) - box.space._queue_shared_sessions:create_index('active', { - type = 'tree', - parts = { 3, 'boolean', 1, str_type() }, - unique = true - }) - else - switch_in_replicaset(queue_session.cfg['in_replicaset']) - - -- At the start of the queue, we make all active sessions inactive, - -- and set an expiration time for them. If the ttr setting is not set, - -- then we delete all sessions. - local ttr = queue_session.cfg['ttr'] or 0 - if ttr > 0 then - for _, tuple in box.space._queue_shared_sessions.index.active:pairs{true} do - box.space._queue_shared_sessions:update(tuple[1], { - {'=', 2, util.event_time(ttr)}, - {'=', 3, false}, - }) - end - else - if queue_session._on_session_remove ~= nil then - for _, tuple in box.space._queue_shared_sessions.index.uuid:pairs() do - queue_session._on_session_remove(tuple[1]) - end - end - box.space._queue_shared_sessions:truncate() - end - end -end - -local function cleanup_inactive_sessions() - local cur_time = util.time() - - for _, val in box.space._queue_shared_sessions.index.active:pairs{false} do - local session_uuid = val[1] - local exp_time = val[2] - if cur_time >= exp_time then - if queue_session._on_session_remove ~= nil then - queue_session._on_session_remove(session_uuid) - end - box.space._queue_shared_sessions:delete{session_uuid} - end - end -end - ---- Create an expiration fiber to cleanup expired sessions. -local function create_expiration_fiber() - local exp_fiber = fiber.create(function() - fiber.self():name('queue_expiration_fiber') - while true do - if box.info.ro == false then - local status, err = pcall(cleanup_inactive_sessions) - if status == false then - log.error('An error occurred while cleanup the sessions: %s', - err) - end - end - if queue_session.sync_chan:get(1) ~= nil then - log.info("Queue expiration fiber was stopped") - break - end - end - end) - - return exp_fiber -end - ---- Identifies the connection and return the UUID of the current session. --- If session_uuid ~= nil: associate the connection with given session. --- In case of attempt to use an invalid format UUID or expired UUID, --- an error will be thrown. -local function identify(conn_id, session_uuid) - local queue_session_ids = box.space._queue_session_ids - local session_ids = queue_session_ids:get(conn_id) - local cur_uuid = session_ids and session_ids[2] - - if session_uuid == nil and cur_uuid ~= nil then - -- Just return the UUID of the current session. - return cur_uuid - elseif session_uuid == nil and cur_uuid == nil then - -- Generate new UUID for the session. - cur_uuid = uuid.bin() - queue_session_ids:insert{conn_id, cur_uuid} - box.space._queue_shared_sessions:insert{cur_uuid, 0, true} - elseif session_uuid ~= nil then - -- Validate UUID. - if not pcall(uuid.frombin, session_uuid) then - error('Invalid UUID format.') - end - - -- identify using a previously created session. - -- Check that a session with this uuid exists. - local ids_by_uuid = queue_session_ids.index.uuid:select( - session_uuid, { limit = 1 })[1] - local shared_session = box.space._queue_shared_sessions:get(session_uuid) - if ids_by_uuid == nil and shared_session == nil then - error('The UUID ' .. uuid.frombin(session_uuid):str() .. - ' is unknown.') - end - - if cur_uuid ~= session_uuid then - if cur_uuid ~= nil then - queue_session.disconnect(conn_id) - end - queue_session_ids:insert({conn_id, session_uuid}) - cur_uuid = session_uuid - end - - -- Make session active. - if shared_session then - box.space._queue_shared_sessions:update(shared_session[1], { - {'=', 2, 0}, - {'=', 3, true}, - }) - end - end - - return cur_uuid -end - -local function on_session_remove(callback) - if type(callback) ~= 'function' then - error('The "on_session_remove" argument type must be "function"') - end - queue_session._on_session_remove = callback -end - ---- Remove a connection from the list of active connections and --- release its tasks if necessary. -local function disconnect(conn_id) - local queue_session_ids = box.space._queue_session_ids - local session_uuid = queue_session.identify(conn_id) - - queue_session_ids:delete{conn_id} - local session_ids = queue_session_ids.index.uuid:select(session_uuid, - { limit = 1 })[1] - - -- If a queue session doesn't have any active connections it should be - -- removed (if ttr is absent) or change the `active` flag to make the - -- session inactive. - if session_ids == nil then - local ttr = queue_session.cfg['ttr'] or 0 - if ttr > 0 then - local tuple = box.space._queue_shared_sessions:get{session_uuid} - if tuple == nil then - box.space._queue_shared_sessions:insert{ - session_uuid, util.event_time(ttr), false - } - else - box.space._queue_shared_sessions:update(tuple[1], { - {'=', 2, util.event_time(ttr)}, - {'=', 3, false}, - }) - end - elseif queue_session._on_session_remove ~= nil then - queue_session._on_session_remove(session_uuid) - box.space._queue_shared_sessions.index.uuid:delete{session_uuid} - end - end -end - -local function grant(user) - box.schema.user.grant(user, 'read, write', 'space', '_queue_session_ids', - { if_not_exists = true }) - box.schema.user.grant(user, 'read, write', 'space', '_queue_shared_sessions', - { if_not_exists = true }) -end - -local function start() - identification_init() - queue_session.sync_chan = fiber.channel() - queue_session.expiration_fiber = create_expiration_fiber() -end - --- When switching the master to the replica, --- the expiration_fiber must be stopped. -local function stop() - queue_session.sync_chan:put(true) -end - -local function validate_opts(opts) - for key, val in pairs(opts) do - if key == 'ttr' then - if type(val) ~= 'number' or val < 0 then - error('Invalid value of ttr: ' .. tostring(val)) - end - elseif key == 'in_replicaset' then - -- Do nothing. - else - error('Unknown option ' .. tostring(key)) - end - end -end - ---- Configure of queue_session module. --- If an invalid value or an unknown option --- is used, an error will be thrown. -local function cfg(self, opts) - opts = opts or {} - -- Check all options before configuring so that - -- the configuration is done transactionally. - validate_opts(opts) - - for key, val in pairs(opts) do - self[key] = val - end - - switch_in_replicaset(self['in_replicaset']) -end - -local function exist_shared(session_uuid) - if box.space._queue_shared_sessions:get{session_uuid} then - return true - end - - return false -end - -queue_session.cfg = setmetatable({}, { __call = cfg }) - --- methods -local method = { - identify = identify, - disconnect = disconnect, - grant = grant, - on_session_remove = on_session_remove, - start = start, - stop = stop, - exist_shared = exist_shared -} - -return setmetatable(queue_session, { __index = method }) diff --git a/queue/abstract/queue_state.lua b/queue/abstract/queue_state.lua deleted file mode 100644 index 289a7d36..00000000 --- a/queue/abstract/queue_state.lua +++ /dev/null @@ -1,135 +0,0 @@ --- Queue states. - -local log = require('log') -local fiber = require('fiber') - ---[[ States switching scheme: - - +-----------+ - | RUNNING | - +-----------+ - ^ | (rw -> ro) - (ro -> rw) | v -+------+ +---------+ +--------+ -| INIT | ---> | STARTUP | | ENDING | -+------+ +---------+ +--------+ - ^ | - (ro -> rw) | v - +-----------+ - | WAITING | - +-----------+ -]]-- - -local queue_state = { - states = { - INIT = 'i', - STARTUP = 's', - RUNNING = 'r', - ENDING = 'e', - WAITING = 'w', - } -} - -local current = queue_state.states.INIT - -local function get_state_key(value) - for k, v in pairs(queue_state.states) do - if v == value then - return k - end - end - - return nil -end - -local function max_lag() - local max_lag = 0 - local n_replica = table.maxn(box.info.replication) - - for i = 1, n_replica do - local replica = box.info.replication[i] - if replica and replica.upstream then - local lag = replica.upstream.lag - if lag > max_lag then - max_lag = lag - end - end - end - - return max_lag -end - -local function create_state_fiber(on_state_change_cb) - log.info('Started queue state fiber') - - fiber.create(function() - fiber.self():name('queue_state_fiber') - while true do - if current == queue_state.states.WAITING then - local rc = pcall(box.ctl.wait_rw) - if rc then - current = queue_state.states.STARTUP - log.info('Queue state changed: STARTUP') - -- Wait for maximum upstream lag * 2. - fiber.sleep(max_lag() * 2) - on_state_change_cb(current) - current = queue_state.states.RUNNING - log.info('Queue state changed: RUNNING') - end - elseif current == queue_state.states.RUNNING then - local rc = pcall(box.ctl.wait_ro) - if rc then - current = queue_state.states.ENDING - on_state_change_cb(current) - log.info('Queue state changed: ENDING') - current = queue_state.states.WAITING - log.info('Queue state changed: WAITING') - end - end - -- Handle server shutdown. - if not pcall(fiber.testcancel) then - break - end - end - end) - - log.info('Finished queue state fiber') -end - --- Public methods. -local method = {} - --- Initialise queue states. Must be called on queue starts. -function method.init(tubes) - current = queue_state.states.RUNNING - create_state_fiber(tubes) -end - --- Show current state in human readable format. -function method.show() - return get_state_key(current) -end - --- Get current state. -function method.get() - return current -end - --- Polls queue state with maximum attemps number. -function method.poll(state, max_attempts) - local attempt = 0 - while true do - if current == state then - return true - end - attempt = attempt + 1 - if attempt == max_attempts then - break - end - fiber.sleep(0.01) - end - - return false -end - -return setmetatable(queue_state, { __index = method }) diff --git a/queue/abstract/state.lua b/queue/abstract/state.lua deleted file mode 100644 index 837dadf9..00000000 --- a/queue/abstract/state.lua +++ /dev/null @@ -1,8 +0,0 @@ --- task states -return { - READY = 'r', - TAKEN = 't', - DONE = '-', - BURIED = '!', - DELAYED = '~', -} diff --git a/queue/compat.lua b/queue/compat.lua deleted file mode 100644 index 3ed6d2bf..00000000 --- a/queue/compat.lua +++ /dev/null @@ -1,127 +0,0 @@ -local fun = require('fun') -local log = require('log') -local json = require('json') -local fiber = require('fiber') - -local iter, op = fun.iter, fun.operator - -local function split(self, sep) - local sep, fields = sep or ":", {} - local pattern = string.format("([^%s]+)", sep) - self:gsub(pattern, function(c) table.insert(fields, c) end) - return fields -end - -local function reducer(res, l, r) - if res ~= nil then - return res - end - if tonumber(l) == tonumber(r) then - return nil - end - return tonumber(l) > tonumber(r) -end - -local function split_version(version_string) - local vtable = split(version_string, '.') - local vtable2 = split(vtable[3], '-') - vtable[3], vtable[4] = vtable2[1], vtable2[2] - return vtable -end - -local function check_version(expected, version) - version = version or _TARANTOOL - if type(version) == 'string' then - version = split_version(version) - end - local res = iter(version):zip(expected):reduce(reducer, nil) - if res or res == nil then res = true end - return res -end - -local function get_actual_numtype(version) - return check_version({1, 7, 2}, version) and 'unsigned' or 'num' -end - -local function get_actual_strtype(version) - return check_version({1, 7, 2}, version) and 'string' or 'str' -end - -local function get_actual_vinylname(version) - return check_version({1, 7}, version) and 'vinyl' or nil -end - -local function get_optname_snapdir(version) - return check_version({1, 7}, version) and 'memtx_dir' or 'snap_dir' -end - -local function get_optname_logger(version) - return check_version({1, 7}, version) and 'log' or 'logger' -end - -local function pack_args(...) - return check_version({1, 7}) and { ... } or ... -end - -local waiter_list = {} - -local function waiter_new() - return setmetatable({ - cond = fiber.cond() - }, { - __index = { - wait = function(self, timeout) - self.cond:wait(timeout) - end, - signal = function(self, wfiber) - self.cond:signal() - end, - free = function(self) - if #waiter_list < 100 then - table.insert(waiter_list, self) - end - end - } - }) -end - -local function waiter_old() - return setmetatable({}, { - __index = { - wait = function(self, timeout) - fiber.sleep(timeout) - end, - signal = function(self, fid) - local wfiber = fiber.find(fid) - if wfiber ~= nil and - wfiber:status() ~= 'dead' and - wfiber:id() ~= fiber.id() then - wfiber:wakeup() - end - end, - free = function(self) - if #waiter_list < 100 then - table.insert(waiter_list, self) - end - end - } - }) -end - -local waiter_actual = check_version({1, 7, 2}) and waiter_new or waiter_old - -local function waiter() - return table.remove(waiter_list) or waiter_actual() -end - -return { - split_version = split_version, - check_version = check_version, - vinyl_name = get_actual_vinylname, - num_type = get_actual_numtype, - str_type = get_actual_strtype, - snapdir_optname = get_optname_snapdir, - logger_optname = get_optname_logger, - pack_args = pack_args, - waiter = waiter -} diff --git a/queue/init.lua b/queue/init.lua deleted file mode 100644 index 6e61faa3..00000000 --- a/queue/init.lua +++ /dev/null @@ -1,166 +0,0 @@ -local fiber = require('fiber') - -local abstract = require('queue.abstract') -local queue_state = require('queue.abstract.queue_state') -local qc = require('queue.compat') -local queue = nil - --- load all core drivers -local core_drivers = { - fifo = require('queue.abstract.driver.fifo'), - fifottl = require('queue.abstract.driver.fifottl'), - utube = require('queue.abstract.driver.utube'), - utubettl = require('queue.abstract.driver.utubettl'), - limfifottl = require('queue.abstract.driver.limfifottl') -} - --- since: --- https://github.com/locker/tarantool/commit/8cf5151cb4f05cee3fd0ea831add2b3187a01fe4 -local watchers_supported = qc.check_version({2, 10, 0}) - -local function register_driver(driver_name, tube_ctr) - if type(tube_ctr.create_space) ~= 'function' or - type(tube_ctr.new) ~= 'function' then - error('tube control methods must contain functions "create_space"' - .. ' and "new"') - end - if queue.driver[driver_name] then - error(('overriding registered driver "%s"'):format(driver_name)) - end - queue.driver[driver_name] = tube_ctr -end - -local deferred_opts = {} - --- We cannot call queue.cfg() while tarantool is in read_only mode. --- This method stores settings for later original queue.cfg() call. -local function deferred_cfg(opts) - opts = opts or {} - - for k, v in pairs(opts) do - deferred_opts[k] = v - end -end - -queue = setmetatable({ - driver = core_drivers, - register_driver = register_driver, - state = queue_state.show, - cfg = deferred_cfg, - _VERSION = require('queue.version'), -}, { __index = function(self, key) - -- In replicaset mode, the user can attempt to call public methods on the replica start. - -- For example, a single script is used for master and replica. - -- Each public method has a check on the state of the queue, so this forwarding is safe. - if deferred_opts['in_replicaset'] == true then - if abstract[key] ~= nil then - return abstract[key] - end - else - print(debug.traceback()) - error('Please configure box.cfg{} in read/write mode first') - end - end -}) - --- Used to store the original methods -local orig_cfg = nil -local orig_call = nil - -local wrapper_impl, handle_instance_mode - -local function rw_waiter() - fiber.name('queue instance rw waiter') - local wait_cond = fiber.cond() - local w = box.watch('box.status', function(_, new_status) - if new_status.is_ro == false then - wait_cond:signal() - end - end) - wait_cond:wait() - w:unregister() - handle_instance_mode() -end - -local function cfg_wrapper(...) - box.cfg = orig_cfg - return wrapper_impl(...) -end - -local function cfg_call_wrapper(cfg, ...) - local cfg_mt = getmetatable(box.cfg) - cfg_mt.__call = orig_call - return wrapper_impl(...) -end - -local function wrap_box_cfg() - if type(box.cfg) == 'function' then - -- box.cfg before the first box.cfg call - orig_cfg = box.cfg - box.cfg = cfg_wrapper - elseif type(box.cfg) == 'table' then - if box.info.ro_reason == 'config' or not watchers_supported then - -- box.cfg after the first box.cfg call. - -- The another call could switch the mode. - local cfg_mt = getmetatable(box.cfg) - orig_call = cfg_mt.__call - cfg_mt.__call = cfg_call_wrapper - else - -- Wait for the rw state. - fiber.new(rw_waiter) - end - else - error('The box.cfg type is unexpected: ' .. type(box.cfg)) - end -end - -function handle_instance_mode() - if box.info.ro == false then - local abstract = require 'queue.abstract' - for name, val in pairs(abstract) do - rawset(queue, name, val) - end - abstract.driver = queue.driver - -- Now the "register_driver" method from abstract will be used. - queue.register_driver = nil - setmetatable(queue, getmetatable(abstract)) - queue.cfg(deferred_opts) - queue.start() - else - -- Delay a start until the box will be configured - -- with read_only = false - wrap_box_cfg() - end -end - -function wrapper_impl(...) - local result = { pcall(box.cfg,...) } - if result[1] then - table.remove(result, 1) - else - wrap_box_cfg() - error(result[2]) - end - - handle_instance_mode() - return unpack(result) -end - ---- Implementation of the “lazy start” procedure. --- The queue module is loaded immediately if the instance was --- configured with read_only = false. Otherwise, a start is --- delayed until the instance will be configured with read_only = false. -local function queue_init() - if rawget(box, 'space') ~= nil and box.info.ro == false then - -- The box was configured with read_only = false - queue = require('queue.abstract') - queue.driver = core_drivers - queue.start() - else - wrap_box_cfg() - end -end - -queue_init() - -return queue diff --git a/queue/util.lua b/queue/util.lua deleted file mode 100644 index 84b350ac..00000000 --- a/queue/util.lua +++ /dev/null @@ -1,48 +0,0 @@ -local fiber = require('fiber') - --- MAX_TIMEOUT == 100 years -local MAX_TIMEOUT = 365 * 86400 * 100 --- Set to TIMEOUT_INFINITY --- instead returns time for next event -local TIMEOUT_INFINITY = 18446744073709551615ULL - ---- Convert seconds to microseconds. --- If tm == nil then returns current system time --- (in microseconds since the epoch). --- If tm < 0 return 0. -local function time(tm) - if tm == nil then - tm = fiber.time64() - elseif tm < 0 then - tm = 0 - else - tm = tm * 1000000 - end - return 0ULL + tm -end - ---- Calculates the system time (in microseconds) of an event that --- will occur after a given time period(tm, specified in seconds). --- If tm <= 0 then returns current system time. -local function event_time(tm) - if tm == nil or tm < 0 then - tm = 0 - elseif tm > MAX_TIMEOUT then - return TIMEOUT_INFINITY - end - tm = 0ULL + tm * 1000000 + fiber.time64() - return tm -end - -local util = { - MAX_TIMEOUT = MAX_TIMEOUT, - TIMEOUT_INFINITY = TIMEOUT_INFINITY -} - --- methods -local method = { - time = time, - event_time = event_time -} - -return setmetatable(util, { __index = method }) diff --git a/queue/version.lua b/queue/version.lua deleted file mode 100644 index 3ef2a2f6..00000000 --- a/queue/version.lua +++ /dev/null @@ -1,4 +0,0 @@ --- Сontains the module version. --- Requires manual update in case of release commit. - -return '1.4.4' diff --git a/rpm/prebuild.sh b/rpm/prebuild.sh deleted file mode 100755 index b380f95b..00000000 --- a/rpm/prebuild.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -set -exu # Strict shell (w/o -o pipefail) - -curl -LsSf https://www.tarantool.io/release/1.10/installer.sh | sudo bash diff --git a/rpm/tarantool-queue.spec b/rpm/tarantool-queue.spec deleted file mode 100644 index 6ccb14b1..00000000 --- a/rpm/tarantool-queue.spec +++ /dev/null @@ -1,51 +0,0 @@ -Name: tarantool-queue -Version: 1.0.1 -Release: 1%{?dist} -Summary: Persistent in-memory queues for Tarantool -Group: Applications/Databases -License: BSD -URL: https://github.com/tarantool/queue -Source0: https://github.com/tarantool/%{name}/archive/%{version}/%{name}-%{version}.tar.gz -BuildArch: noarch -BuildRequires: tarantool >= 1.7 -BuildRequires: tarantool-devel >= 1.7 -BuildRequires: /usr/bin/prove -Requires: tarantool >= 1.7 -%description -A collection of persistent queue implementations for Tarantool. - -%prep -%setup -q -n %{name}-%{version} - -%build -%cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo - -# %check -# make check -# -%define luapkgdir %{_datadir}/tarantool/queue/ -%install -%if 0%{?fedora} >= 33 || 0%{?rhel} >= 8 - %cmake_install -%else - %make_install -%endif - -%files -%dir %{luapkgdir} -%{luapkgdir}/*.lua -%{luapkgdir}/abstract/*.lua -%{luapkgdir}/abstract/driver/*.lua -%doc README.md -%{!?_licensedir:%global license %doc} -%license LICENSE - -%changelog -* Wed Apr 18 2018 Alexander Turenko 1.0.1 -- Update tarantool dependency to 1.7 - -* Thu Apr 06 2016 Eugene Blikh 1.0.1-6 -- RPM spec uses CMake now (depend on tarantool0devel) - -* Thu Feb 18 2016 Roman Tsisyk 1.0.0-1 -- Initial version of the RPM spec diff --git a/t/000-init.t b/t/000-init.t deleted file mode 100755 index ee7b9e21..00000000 --- a/t/000-init.t +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env tarantool -local yaml = require('yaml') -local fiber = require('fiber') - -local test = require('tap').test() -test:plan(2) - -local queue = require('queue') - -local engine = os.getenv('ENGINE') or 'memtx' - -test:test('access to queue until box.cfg is started', function(test) - test:plan(3) - test:isnil(rawget(box, 'space'), 'box is not started yet') - - local s, e = pcall(function() return queue.tube end) - test:ok(not s, 'exception was generated') - test:ok(string.match(e, 'Please configure box.cfg') ~= nil, 'Exception text') -end) - -local state = require('queue.abstract.state') - -local tnt = require('t.tnt') -tnt.cfg{} - -test:test('access to queue after box.cfg{}', function(test) - test:plan(9) - test:istable(queue.tube, 'queue.tube is table') - test:is(#queue.tube, 0, 'queue.tube is empty') - - local tube = queue.create_tube('test', 'fifo', { engine = engine }) - test:ok(queue.tube.test, 'tube "test" is created') - - test:ok(queue.tube.test:put(123), 'put') - - local task = queue.tube.test:take(.1) - test:ok(task, 'task was taken') - test:is(task[3], 123, 'task.data') - test:ok(queue.tube.test:ack(task[1]), 'task.ack') - - test:ok(queue.tube.test:drop(), 'tube.drop') - test:isnil(queue.tube.test, 'tube is really removed') -end) - -tnt.finish() -os.exit(test:check() and 0 or 1) --- vim: set ft=lua : diff --git a/t/000-queue.t b/t/000-queue.t new file mode 100644 index 00000000..2d02b9a7 --- /dev/null +++ b/t/000-queue.t @@ -0,0 +1,503 @@ +#!/usr/bin/perl + +use warnings; +use strict; +use utf8; +use open qw(:std :utf8); +use lib qw(lib ../lib); + +use Test::More; +use constant PLAN => 81; + +BEGIN { + system 'which tarantool_box >/dev/null 2>&1'; + if ($? == 0) { + plan tests => PLAN; + } else { + plan skip_all => 'tarantool_box not found'; + } +} + +use Encode qw(decode encode); +use Cwd 'cwd'; +use File::Spec::Functions 'catfile'; +# use feature 'state'; + + + +BEGIN { + # Подготовка объекта тестирования для работы с utf8 + my $builder = Test::More->builder; + binmode $builder->output, ":utf8"; + binmode $builder->failure_output, ":utf8"; + binmode $builder->todo_output, ":utf8"; + + use_ok 'Coro'; + use_ok 'DR::Tarantool', ':all'; + use_ok 'DR::Tarantool::StartTest'; + use_ok 'Time::HiRes', 'time'; + use_ok 'Coro::AnyEvent'; +} +my $t = DR::Tarantool::StartTest->run( + cfg => catfile(cwd, 'tarantool.cfg'), + script_dir => catfile(cwd) +); + +$SIG{INT} = sub { + note $t->log if $ENV{DEBUG}; + $t->kill('KILL'); + exit 2; +}; + +our $tnt; +sub tnt { + unless($tnt) { + $tnt = eval { + coro_tarantool + host => 'localhost', + port => $t->primary_port, + spaces => { + 0 => { + name => 'queue', + default_type => 'STR', + fields => [ + qw(uuid tube status), + { + type => 'NUM64', + name => 'event' + }, + { + type => 'STR', + name => 'ipri' + }, + { + type => 'STR', + name => 'pri' + }, + { + type => 'NUM', + name => 'cid', + }, + { + type => 'NUM64', + name => 'started' + }, + { + type => 'NUM64', + name => 'ttl', + }, + { + type => 'NUM64', + name => 'ttr', + }, + { + type => 'NUM64', + name => 'bury' + }, + { + type => 'NUM64', + name => 'taken' + }, + 'task', + ], + indexes => { + 0 => 'uuid', + 1 => { + name => 'event', + fields => [qw(tube status event pri)] + } + } + } + }, + }; + note $t->log unless $tnt; + } + $tnt; +}; + +ok tnt->ping, 'ping tarantool'; +diag $t->log unless + ok $t->started, 'Tarantool was started'; +diag $t->log unless + ok eval { tnt }, 'Client connected to'; + +my $sno = tnt->space('queue')->number; + +my $task1 = tnt->call_lua('queue.put', + [ + $sno, + 'tube_name', + 0, + 10, + 20, + 30, + 'task', 1 .. 10 + ] +)->raw; + + + +is tnt->call_lua('queue.meta', [ $sno, $task1->[0] ])->raw(2), 'ready', + 'task1 is ready'; + +my $meta = tnt->call_lua('queue.meta', [ $sno, $task1->[0] ], + fields => [ + { type => 'STR', name => 'id' }, + { type => 'STR', name => 'tube' }, + { type => 'STR', name => 'status' }, + { type => 'NUM64', name => 'event' }, + { type => 'STR', name => 'ipri' }, + { type => 'STR', name => 'pri' }, + { type => 'NUM', name => 'cid' }, + { type => 'NUM64', name => 'created' }, + { type => 'NUM64', name => 'ttl' }, + { type => 'NUM64', name => 'ttr' }, + { type => 'NUM', name => 'cbury' }, + { type => 'NUM', name => 'ctaken' }, + { type => 'NUM64', name => 'now' }, + ] +); + +cmp_ok $meta->now/1000000, '<', $meta->ttl + $meta->created, 'task is alive'; +is $meta->ttl, 10, 'ttl'; +is $meta->ttr, 20, 'ttr'; + +is_deeply $task1, [ $task1->[0], 'tube_name', 'ready', 'task', 1 .. 10 ], + 'task 1'; + + +my $started = time; +my $task2 = tnt->call_lua('queue.put', + [ + $sno, + 'tube_name', + .5, + 10, + 20, + 30, + 'task', 10 .. 20 + ] +)->raw; + +is tnt->call_lua('queue.meta', [ $sno, $task2->[0] ])->raw(2), 'delayed', + 'task2 is delayed'; + +is_deeply tnt->call_lua('queue.peek', [ $sno, $task2->[0] ])->raw, $task2, + 'task2.get'; + +is_deeply $task2, [ $task2->[0], 'tube_name', + 'delayed', 'task', 10 .. 20 ], 'task 2'; + +my $task1_t = tnt->call_lua('queue.take', [ $sno, 'tube_name', 5 ])->raw; +$task1->[2] = 'taken'; +is_deeply $task1_t, $task1, 'task1 taken'; +is tnt->call_lua('queue.meta', [ $sno, $task1->[0] ])->raw(2), 'taken', + 'task1 is taken'; + + +my $task2_t = eval {tnt->call_lua('queue.take', [ $sno, 'tube_name', 5 ])->raw}; +$task2->[2] = 'taken'; +is_deeply $task2_t, $task2, 'task2 taken'; +cmp_ok time - $started, '>=', .5, 'delay more than 0.5 second'; +cmp_ok time - $started, '<=', .7, 'delay less than 0.7 second'; + +is_deeply tnt->call_lua('queue.peek', [ $sno, $task2->[0] ])->raw, $task2, + 'queue.peek'; + +my $task_ack = tnt->call_lua('queue.ack', [ $sno, $task2->[0] ])->raw; +is_deeply $task_ack, $task2, 'task was ack'; + + +is_deeply scalar eval { tnt->call_lua('queue.peek', [ $sno, $task2->[0] ]) }, + undef, 'queue.peek'; +like $@, qr{Task not found}, 'task not found'; + +$task_ack = eval { tnt->call_lua('queue.ack', [ $sno, $task2->[0] ]) }; +like $@, qr{task not found}i, 'error message'; +is $task_ack, undef, 'repeat ack'; + +$task1->[2] = 'ready'; +is_deeply tnt->call_lua('queue.release', [ $sno, $task1->[0] ])->raw, $task1, + 'task1 release'; +is tnt->call_lua('queue.meta', [ $sno, $task1->[0] ])->raw(2), 'ready', + 'task1 is ready'; + +$task1_t = tnt->call_lua('queue.take', [ $sno, 'tube_name', 5 ])->raw; +$task1->[2] = 'taken'; +is_deeply $task1_t, $task1, 'repeatly take task1'; +$started = time; +$task1->[2] = 'delayed'; +is_deeply tnt->call_lua('queue.release', [ $sno, $task1->[0], .5 ])->raw, + $task1, 'task1 release (delayed)'; +is tnt->call_lua('queue.meta', [ $sno, $task1->[0] ])->raw(2), 'delayed', + 'task1 is delayed'; +$task1_t = tnt->call_lua('queue.take', [ $sno, 'tube_name', 5 ])->raw; +cmp_ok time - $started, '>=', .5, 'take took more than 0.5 second'; +cmp_ok time - $started, '<=', .7, 'take took less than 0.7 second'; + +is tnt->call_lua('queue.meta', [ $sno, $task1->[0] ])->raw(2), 'taken', + 'task1 is run'; +$task1->[2] = 'taken'; +is_deeply $task1_t, $task1, 'task1 is deeply'; + +is_deeply tnt->call_lua('queue.ack', [ $sno, $task1->[0]])->raw, + $task1, 'task1.ack'; + + +$task1 = tnt->call_lua('queue.put', + [ + $sno, + 'tube_name', + 0, # delay + 5, # ttl + 0.5, # ttr + 30, # pri + 'task', 30 .. 40 + ] +)->raw; + +my $task1_m = tnt->call_lua('queue.meta', [ $sno, $task1->[0] ], 'queue'); + +$task2 = tnt->call_lua('queue.urgent', + [ + $sno, + 'tube_name', + 0, # delay + 5, # ttl + 0.5, # ttr + 30, # pri + 'task', 40 .. 50 + ] +)->raw; + +my $task3 = tnt->call_lua('queue.put', + [ + $sno, + 'tube_name', + .5, # delay + .1, # ttl + 0.1, # ttr + 30, # pri + 'task', 50 .. 60 + ] +)->raw; + +my $task4 = tnt->call_lua('queue.put', + [ + $sno, + 'tube_name', + 0, # delay + 5, # ttl + 1, # ttr + 30, # pri + 'task', 50 .. 60 + ] +)->raw; + +my $task2_m = tnt->call_lua('queue.meta', [ $sno, $task2->[0] ], 'queue'); + +cmp_ok $task2_m->ipri, '<', $task1_m->ipri, 'ipri(task2) < ipri(task1)'; + +$task2_t = tnt->call_lua('queue.take', [ $sno, 'tube_name' ])->raw; +$task1_t = tnt->call_lua('queue.take', [ $sno, 'tube_name' ])->raw; +isnt $task1->[0], $task2->[0], "task1 and task2 aren't the same"; + +$task1->[2] = $task2->[2] = 'taken'; +is_deeply $task1_t, $task1, 'task1 was fetched as urgent'; +is_deeply $task2_t, $task2, 'task2 was fetched as usual'; + + +for (1 .. 10) { + Coro::AnyEvent::sleep .2; + tnt->call_lua('queue.touch', [ $sno, $task1->[0] ])->raw; +} + +$task1_m = tnt->call_lua('queue.meta', [ $sno, $task1->[0] ], 'queue'); +$task2_m = tnt->call_lua('queue.meta', [ $sno, $task2->[0] ], 'queue'); + +is $task1_m->status, 'taken', 'task1 is taken'; +is $task2_m->status, 'ready', 'task2 is ready (by ttr)'; + +is scalar eval { tnt->call_lua('queue.peek', [ $sno, $task3->[0] ]) }, undef, + 'task3 was deleted by ttl'; +like $@, qr{Task not found}, 'error string'; + +$task1->[2] = 'ready'; +is_deeply tnt->call_lua('queue.requeue', [ $sno, $task1->[0] ])->raw, + $task1, 'task1.requeue'; + +cmp_ok tnt->call_lua('queue.meta', [ $sno, $task1->[0] ], 'queue')->ipri, + '>', + $task1_m->ipri, + 'requeue increase ipri'; + +$task1_m = tnt->call_lua('queue.meta', [ $sno, $task1->[0] ], 'queue'); +is $task1_m->status, 'ready', 'requeue sets status as ready'; +is $task1_m->taken, 1, 'task has ctaken=1'; + +$task1 = tnt->call_lua('queue.take', [ $sno, 'tube_name' ])->raw; +$task2 = tnt->call_lua('queue.take', [ $sno, 'tube_name' ])->raw; +isnt $task1->[0], $task2->[0], 'task1 and task2 were taken'; + +$tnt->disconnect(sub { }); +$tnt = undef; + +ok tnt->ping, 'new tarantool connection'; + +$task1_m = tnt->call_lua('queue.meta', [ $sno, $task1->[0] ], 'queue'); +$task2_m = tnt->call_lua('queue.meta', [ $sno, $task2->[0] ], 'queue'); + +is $task1_m->status, 'ready', 'task1 is ready (by dead consumer)'; +is $task2_m->status, 'ready', 'task2 is ready (by dead consumer)'; + +$task1 = tnt->call_lua('queue.take', [ $sno, 'tube_name' ])->raw; +$task2 = tnt->call_lua('queue.take', [ $sno, 'tube_name' ])->raw; +isnt $task1->[0], $task2->[0], 'task1 and task2 were taken'; + +$task1_t = tnt->call_lua('queue.done', [ $sno, $task1->[0], 'abc' ])->raw; + +is_deeply + $task1_t, + [ $task1->[0], 'tube_name', 'done', 'abc' ], + 'task1.done'; + +$task1_m = tnt->call_lua('queue.meta', [ $sno, $task1->[0] ], 'queue'); +is $task1_m->status, 'done', 'task1.status'; + +my %stat = @{ tnt->call_lua('queue.statistics', [])->raw }; + +is $stat{"space$sno.tube_name.ready_by_disconnect"}, 2, + 'cnt ready_by_disconnect'; +is $stat{"space$sno.tube_name.ttl.total"}, 1, + 'cnt ttl.total'; +is $stat{"space$sno.tube_name.tasks.buried"}, 0, + 'cnt tasks.buried'; +is $stat{"space$sno.tube_name.tasks.ready"}, 1, 'cnt tasks.ready'; +is $stat{"space$sno.tube_name.tasks.taken"}, 1, 'cnt tasks.taken'; +is $stat{"space$sno.tube_name.tasks.done"}, 1, 'cnt tasks.done'; + + +$task2_t = tnt->call_lua('queue.bury', [ $sno, $task2->[0] ])->raw; +$task2->[2] = 'buried'; +is_deeply $task2_t, $task2, 'task2.bury'; +$task2_m = tnt->call_lua('queue.meta', [ $sno, $task2->[0] ], 'queue'); +is $task2_m->status, 'buried', 'task2.status'; + +is scalar eval { tnt->call_lua('queue.dig', [ $sno, $task1->[0] ])}, undef, + 'wrong task.dig'; + +$task2_t = tnt->call_lua('queue.dig', [ $sno, $task2->[0] ])->raw; +$task2->[2] = 'ready'; +is_deeply $task2_t, $task2, 'task2.dig'; +$task2_m = tnt->call_lua('queue.meta', [ $sno, $task2->[0] ], 'queue'); +is $task2_m->status, 'ready', 'task2.status'; + + +$task1 = tnt->call_lua('queue.take', [ $sno, 'tube_name' ])->raw; +$task2 = tnt->call_lua('queue.take', [ $sno, 'tube_name' ])->raw; + +# test restart function +is tnt->call_lua('queue.restart_check', [ $sno, 'tube_name' ])->raw(0), + 'already started', + 'call restart function'; +tnt->call_lua('box.dostring', [ 'queue.restart = {}' ]); +is tnt->call_lua('queue.restart_check', [ $sno, 'tube_name' ])->raw(0), + 'starting', + 'call restart function'; +is tnt->call_lua('queue.restart_check', [ $sno, 'tube_name' ])->raw(0), + 'already started', + 'call restart function'; + + +isnt $task1->[0], $task2->[0], '2 tasks were taken'; +$task1_t = tnt->call_lua('queue.bury', [ $sno, $task1->[0] ])->raw; +$task2_t = tnt->call_lua('queue.bury', [ $sno, $task2->[0] ])->raw; +$task1_m = tnt->call_lua('queue.meta', [ $sno, $task1->[0] ], 'queue'); +$task2_m = tnt->call_lua('queue.meta', [ $sno, $task2->[0] ], 'queue'); +is $task1_m->status, 'buried', 'task1.status'; +is $task2_m->status, 'buried', 'task2.status'; + + +my $kicked = tnt->call_lua( + 'queue.kick', [ $sno, 'tube_name', 10 ], + fields => [ { name => 'count', type => 'NUM' }] +)->raw(0); + +is $kicked, 2, '2 tasks were kicked'; + +$task1_m = tnt->call_lua('queue.meta', [ $sno, $task1->[0] ], 'queue'); +$task2_m = tnt->call_lua('queue.meta', [ $sno, $task2->[0] ], 'queue'); +is $task1_m->status, 'ready', 'task1.status'; +is $task2_m->status, 'ready', 'task2.status'; + + +$task1 = tnt->call_lua('queue.put', [ $sno, 'pri', 0, 10, 10, 30, 'pri=30']) + ->raw; +$task2 = tnt->call_lua('queue.put', [ $sno, 'pri', 0, 10, 10, 10, 'pri=10']) + ->raw; +$task3 = tnt->call_lua('queue.put', [ $sno, 'pri', 0, 10, 10, 20, 'pri=20']) + ->raw; + + +$task1->[2] = $task2->[2] = $task3->[2] = 'taken'; + + $task1_t = tnt->call_lua('queue.take', [ $sno, 'pri' ])->raw; +my $task3_t = tnt->call_lua('queue.take', [ $sno, 'pri' ])->raw; + $task2_t = tnt->call_lua('queue.take', [ $sno, 'pri' ])->raw; + +is_deeply $task1_t, $task1, 'task1 (pri)'; +is_deeply $task2_t, $task2, 'task2 (pri)'; +is_deeply $task3_t, $task3, 'task3 (pri)'; + +SKIP: { + my $task_unique1 = eval { + tnt->call_lua('queue.put_unique', + [ + $sno, + 'tube_name', + 0, # delay + 5, # ttl + 1, # ttr + 30, # pri + 'unique_task', 50 .. 60 + ] + )->raw; + }; + skip 'index is not configured for put_unique', 2 + if !$task_unique1 and $@ =~ /unique/; + + my $task_unique2 = tnt->call_lua('queue.put_unique', + [ + $sno, + 'tube_name', + 0, # delay + 5, # ttl + 1, # ttr + 30, # pri + 'unique_task', 50 .. 60 + ] + )->raw; + + is_deeply $task_unique1, $task_unique2, 'unique tasks'; + + eval { + tnt->call_lua('queue.put_unique', + [ + $sno, + 'tube_name', + 0, # delay + 5, # ttl + 1, # ttr + 30 # pri + ] + ); + }; + + like $@, qr/Can not put unique task without data/, + 'unique task without data are prohibited'; + +} + +END { + note $t->log if $ENV{DEBUG}; +} diff --git a/t/001-tube-init.t b/t/001-tube-init.t deleted file mode 100755 index c48442d6..00000000 --- a/t/001-tube-init.t +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env tarantool -local yaml = require('yaml') -local fiber = require('fiber') - -local tnt = require('t.tnt') - -local test = require('tap').test() -test:plan(1) - -local engine = os.getenv('ENGINE') or 'memtx' - -local mock_tube = { create_space = function() end, new = function() end } - -test:test('test queue mock addition', function(test) - test:plan(3) - - local queue = require('queue') - queue.register_driver('mock', mock_tube) - test:is(queue.driver.mock, mock_tube) - - local res, err = pcall(queue.register_driver, 'mock', mock_tube) - local check = not res and - string.match(err, 'overriding registered driver') ~= nil - test:ok(check, 'check a driver override failure') - - tnt.cfg{} - - test:is(queue.driver.mock, mock_tube) -end) - -tnt.finish() -os.exit(test:check() and 0 or 1) --- vim: set ft=lua : diff --git a/t/010-dr-tqueue.t b/t/010-dr-tqueue.t new file mode 100644 index 00000000..61f4d864 --- /dev/null +++ b/t/010-dr-tqueue.t @@ -0,0 +1,239 @@ +#!/usr/bin/perl + +use warnings; +use strict; +use utf8; +use open qw(:std :utf8); +use lib qw(lib ../lib); + +use Test::More; +use constant PLAN => 82; + +BEGIN { + system 'which tarantool_box >/dev/null 2>&1'; + if ($? == 0) { + plan tests => PLAN; + } else { + plan skip_all => 'tarantool_box not found'; + } +} + +use Encode qw(decode encode); +use Cwd 'cwd'; +use File::Spec::Functions 'catfile'; +# use feature 'state'; + + + +BEGIN { + # Подготовка объекта тестирования для работы с utf8 + my $builder = Test::More->builder; + binmode $builder->output, ":utf8"; + binmode $builder->failure_output, ":utf8"; + binmode $builder->todo_output, ":utf8"; + + use_ok 'DR::TarantoolQueue'; + use_ok 'Coro'; + use_ok 'DR::Tarantool', ':all'; + use_ok 'DR::Tarantool::StartTest'; + use_ok 'Time::HiRes', 'time'; + use_ok 'Coro::AnyEvent'; +} +my $t = DR::Tarantool::StartTest->run( + cfg => catfile(cwd, 'tarantool.cfg'), + script_dir => catfile(cwd) +); + + +my $q = DR::TarantoolQueue->new( + host => '127.0.0.1', + port => $t->primary_port, + space => 0, + tube => 'test_queue', +); + +my $qs = DR::TarantoolQueue->new( + host => '127.0.0.1', + port => $t->primary_port, + space => 0, + tube => 'test_queue', + coro => 0 +); + +isa_ok $q => 'DR::TarantoolQueue'; + +my (@f, @fh); +push @f => async { + ok $q->tnt->ping, 'tnt ping'; + push @fh => fileno($q->tnt->_llc->{fh}) +} for 1 .. 5; +$_->join for @f; +@f = (); + +if (DR::Tarantool->can('rsync_tarantool')) { + isa_ok $qs->tnt => 'DR::Tarantool::RealSyncClient'; +} else { + isa_ok $qs->tnt => 'DR::Tarantool::SyncClient'; +} + +ok 5 == grep({ $_ == $fh[0] } @fh), 'connection established once'; +ok $qs->tnt->ping, 'ping by sync client'; + +for ('put', 'urgent') { + my $task1 = $q->$_; + is_deeply $task1->data, undef, "$_()"; + like $task1->id, qr[^[0-9a-fA-F]{32}$], 'task1.id'; + my $task2 = $q->$_(data => { 1 => 2 }); + like $task2->id, qr[^[0-9a-fA-F]{32}$], 'task2.id'; + is_deeply $task2->data, { 1 => 2 }, "$_(data => hashref)"; + my $task3 = $q->$_(data => [ 3, 4, 'привет' ]); + like $task3->id, qr[^[0-9a-fA-F]{32}$], 'task3.id'; + is_deeply $task3->data, [ 3, 4, 'привет' ], "$_(data => arrayref)"; + +} + + +my $task1_t = $q->take; +isa_ok $task1_t => 'DR::TarantoolQueue::Task'; +my $task2_t = $q->take; +isa_ok $task2_t => 'DR::TarantoolQueue::Task'; +my $task3_t = $q->take; +isa_ok $task3_t => 'DR::TarantoolQueue::Task'; + + +isnt $task1_t->id, $task2_t->id, "task1 and task2 aren't the same"; +isnt $task1_t->id, $task3_t->id, "task1 and task3 aren't the same"; + + +is $task1_t->status, 'taken', 'task is taken'; +isa_ok $task1_t->ack => 'DR::TarantoolQueue::Task', 'task1.ack'; +is $task1_t->status, 'ack(removed)', 'task is ack'; +isa_ok $q->ack(id => $task2_t->id), 'DR::TarantoolQueue::Task', 'task2.ack'; + +my $meta = $task3_t->get_meta; +isa_ok $meta => 'HASH', 'task3.meta.meta'; +is $meta->{status}, 'taken', 'task3.meta.status'; +is $meta->{ctaken}, 1, 'task3.meta.ctaken'; +is $meta->{cbury}, 0, 'task3.meta.cbury'; +is $meta->{tube}, 'test_queue', 'task3.meta.tube'; +is $meta->{status}, $task3_t->status, 'task3.status'; + +my $stat = $q->statistics; +isa_ok $stat => 'HASH', 'statistics'; +my $stat2 = $q->statistics(space => 0, tube => undef); +is_deeply $stat, $stat2, 'stats are the same'; +$stat2 = $q->statistics(tube => 'test_queue' ); +is_deeply $stat2, $stat, 'stats are the same'; + + +$stat2 = $q->statistics(space => 123); +is_deeply $stat2, {}, 'empty stat'; +$stat2 = $q->statistics(tube => 'abc' ); +is_deeply $stat2, {}, 'empty stat'; + + + + +my $task1 = $q->take; +isa_ok $task1 => 'DR::TarantoolQueue::Task'; +my $task2 = $q->take; +isa_ok $task2 => 'DR::TarantoolQueue::Task'; +my $task3 = $q->take; +isa_ok $task3 => 'DR::TarantoolQueue::Task'; + +$meta = $task2->get_meta; + +is $task1->status, 'taken', 'task1 is taken'; +is $task2->status, 'taken', 'task2 is taken'; +is $task3->status, 'taken', 'task3 is taken'; + +$task1_t = $task1->release(delay => 10); +isa_ok $task1_t => 'DR::TarantoolQueue::Task'; +$task2_t = $task2->release(delay => 20, ttl => 30); +isa_ok $task1_t => 'DR::TarantoolQueue::Task'; +$task3_t = $task3->release; +isa_ok $task1_t => 'DR::TarantoolQueue::Task'; + +is $task1_t->status, 'delayed', 'task1 released as delayed'; +is $task1->status, 'delayed', 'task1 released as delayed'; +is $task2_t->status, 'delayed', 'task2 released as delayed'; +is $task2->status, 'delayed', 'task2 released as delayed'; +is $task3_t->status, 'ready', 'task3 released as ready'; +is $task3->status, 'ready', 'task3 released as ready'; + +cmp_ok $task2->get_meta->{ttl}, '<', $meta->{ttl}, 'release updated ttl'; +cmp_ok $task2->get_meta->{ttl}, '>=', (30+20), + 'ttl is more than 50s'; +cmp_ok $task2->get_meta->{ttl}, '<', (30+30), + 'ttl is less than 60s'; + + +$task1 = $q->take; +isa_ok $task1 => 'DR::TarantoolQueue::Task'; +is $task1->status, 'taken', 'task1 is taken'; +$task1_t = $task1->done(data => {'превед', 'медвед'}); +is $task1->status, 'done', 'task1 is done'; +is_deeply $task1->data, { 'превед', 'медвед' }, 'task1 is done'; +isa_ok $task1_t => 'DR::TarantoolQueue::Task'; +is $task1_t->status, 'done', 'task is done'; +is_deeply $task1_t->data, { 'превед', 'медвед' }, 'task.data'; + + +my $task4 = $q->put(tube => 'utftube', data => [ 3, 4, 'привет' ]); +like $task4->id, qr[^[0-9a-fA-F]{32}$], 'task3.id'; +is_deeply $task4->data, [ 3, 4, 'привет' ], + "put(data => arrayref)"; + +my $task5 = + $q->urgent(tube => 'utftube', data => [ 3, 4, encode utf8 => 'медвед' ]); +like $task5->id, qr[^[0-9a-fA-F]{32}$], 'task3.id'; +is_deeply $task5->data, [ 3, 4, encode utf8 => 'медвед' ], + "urgent(data => arrayref)"; + +my $task5_t = $q->take(tube => 'utftube'); +my $task4_t = $q->take(tube => 'utftube'); + +is_deeply $task4->data, $task4_t->data, 'Task and decoded utf data'; +is_deeply $task5->data, $task5_t->data, 'Task and encoded utf data'; + +SKIP: { + my $task_unique1 = eval { + $q->put_unique(tube => 'utftube_unique', + data => [ 3, 4, 'привет' ]); + }; + skip 'tarantool is not configured for put_unique', 1 + if !$task_unique1 and $@ =~ /put_unique/; + my $task_unique2 = $q->put_unique(tube => 'utftube_unique', + data => [ 3, 4, 'привет' ]); + + is_deeply $task_unique1->id, $task_unique2->id, + 'Unique tasks putting has equal ids'; + +} +{ + use Scalar::Util 'refaddr'; + $q = DR::TarantoolQueue->new( + host => '127.0.0.1', + port => $t->primary_port, + space => 0, + tube => 'test_queue', + coro => 1 + ); + ok $q, 'queue instance is created'; + is $q->{tnt}, undef, 'connection is not created yet'; + + my (@clist, @f); + for (1 .. 100) { + push @f => async { $q->put(data => $_) }; + push @clist => $q->{tnt}; + } + $_->join for @f; + is_deeply [ map { refaddr $_ } @clist ], + [ map { refaddr $clist[0] } 1 .. 100 ], + 'Only one connection established'; + + +} +END { + note $t->log if $ENV{DEBUG}; +} diff --git a/t/010-fifo.t b/t/010-fifo.t deleted file mode 100755 index 834176be..00000000 --- a/t/010-fifo.t +++ /dev/null @@ -1,327 +0,0 @@ -#!/usr/bin/env tarantool -local yaml = require('yaml') -local fiber = require('fiber') - -local test = require('tap').test() -test:plan(15) - -local queue = require('queue') -local state = require('queue.abstract.state') - -local engine = os.getenv('ENGINE') or 'memtx' - -local tnt = require('t.tnt') -tnt.cfg{} - -test:ok(rawget(box, 'space'), 'box started') -test:ok(queue, 'queue is loaded') - -local tube = queue.create_tube('test', 'fifo', { engine = engine }) -local tube2 = queue.create_tube('test_stat', 'fifo', { engine = engine }) -test:ok(tube, 'test tube created') -test:is(tube.name, 'test', 'tube.name') -test:is(tube.type, 'fifo', 'tube.type') - -test:test('Fifo statistics', function(test) - test:plan(13) - tube2:put('stat_0') - tube2:put('stat_1') - tube2:put('stat_2') - tube2:put('stat_3') - tube2:put('stat_4') - tube2:delete(4) - tube2:take(.001) - tube2:release(0) - tube2:take(.001) - tube2:ack(0) - tube2:bury(1) - tube2:bury(2) - tube2:kick(1) - tube2:take(.001) - - local stats = queue.statistics('test_stat') - - -- check tasks statistics - test:is(stats.tasks.taken, 1, 'tasks.taken') - test:is(stats.tasks.buried, 1, 'tasks.buried') - test:is(stats.tasks.ready, 1, 'tasks.ready') - test:is(stats.tasks.done, 2, 'tasks.done') - test:is(stats.tasks.delayed, 0, 'tasks.delayed') - test:is(stats.tasks.total, 3, 'tasks.total') - - -- check function call statistics - test:is(stats.calls.delete, 1, 'calls.delete') - test:is(stats.calls.ack, 1, 'calls.ack') - test:is(stats.calls.take, 3, 'calls.take') - test:is(stats.calls.kick, 1, 'calls.kick') - test:is(stats.calls.bury, 2, 'calls.bury') - test:is(stats.calls.put, 5, 'calls.put') - test:is(stats.calls.release, 1, 'calls.release') -end) - -test:test('Easy put/take/ack', function(test) - test:plan(5) - - test:ok(tube:put{123}, 'task was put') - local task = tube:take() - test:ok(task, 'task was taken') - test:is(task[2], 't', 'task status') - task = tube:ack(task[1]) - test:ok(task, 'task was acked') - test:is(task[2], '-', 'task status') -end) - - -test:test('fifo', function(test) - test:plan(5) - - test:test('order', function(test) - test:plan(16) - test:ok(tube:put('first'), 'put task 1') - test:ok(tube:put(2), 'put task 2') - test:ok(tube:put('third'), 'put task 3') - test:ok(tube:put(4), 'put task 4') - - local task1 = tube:take() - test:ok(task1, 'task 1 was taken') - test:is(task1[2], 't', 'task1 status') - test:is(task1[3], 'first', 'task1.data') - - local task2 = tube:take() - test:ok(task2, 'task 2 was taken') - test:is(task2[2], 't', 'task2 status') - test:is(task2[3], 2, 'task2.data') - - local task3 = tube:take() - test:ok(task3, 'task 3 was taken') - test:is(task3[2], 't', 'task3 status') - test:is(task3[3], 'third', 'task3.data') - - local task4 = tube:take() - test:ok(task4, 'task 4 was taken') - test:is(task4[2], 't', 'task4 status') - test:is(task4[3], 4, 'task4.data') - end) - - test:test('timeouts', function(test) - test:plan(3) - - test:isnil(tube:take(.1), 'task was not taken: timeout') - - fiber.create(function() - test:ok(tube:take(1), 'task was taken') - end) - - fiber.sleep(0.1) - test:ok(tube:put(123), 'task was put') - fiber.sleep(0.1) - end) - - test:test('release', function(test) - test:plan(20) - test:ok(tube:put('first'), 'put task 1') - test:ok(tube:put(2), 'put task 2') - test:ok(tube:put('third'), 'put task 3') - test:ok(tube:put(4), 'put task 4') - - local task1 = tube:take() - test:ok(task1, 'task 1 was taken') - test:is(task1[2], 't', 'task1 status') - test:is(task1[3], 'first', 'task1.data') - - local task2 = tube:take() - test:ok(task2, 'task 2 was taken') - test:is(task2[2], 't', 'task2 status') - test:is(task2[3], 2, 'task2.data') - - - test:ok(tube:release(task1[1]), 'task1 was released') - - local task1a = tube:take() - test:ok(task1a, 'task 1 was taken again') - test:is(task1a[2], 't', 'task1 status') - test:is(task1a[3], 'first', 'task1.data') - - local task3 = tube:take() - test:ok(task3, 'task 3 was taken') - test:is(task3[2], 't', 'task3 status') - test:is(task3[3], 'third', 'task3.data') - - local task4 = tube:take() - test:ok(task4, 'task 4 was taken') - test:is(task4[2], 't', 'task4 status') - test:is(task4[3], 4, 'task4.data') - end) - - test:test('release timeouts', function(test) - test:plan(5) - - test:ok(tube:put('test123'), 'task was put') - local task = tube:take(.1) - test:is(task[3], 'test123', 'task.data') - - fiber.create(function() - local task = tube:take(1) - test:ok(task, 'task was taken') - test:is(task[3], 'test123', 'task.data') - end) - - fiber.sleep(0.1) - test:ok(tube:release(task[1]), 'task was released') - fiber.sleep(0.1) - end) - - - test:test('bury/kick/delete/peek', function(test) - test:plan(18) - test:ok(tube:put('first'), 'put task 1') - test:ok(tube:put(2), 'put task 2') - test:ok(tube:put('third'), 'put task 3') - test:ok(tube:put(4), 'put task 4') - - local task1 = tube:take() - local task2 = tube:take() - local task3 = tube:take() - local task4 = tube:take() - - task1 = tube:bury(task1[1]) - test:is(task1[2], state.BURIED, 'task1 is buried') - test:ok(tube:bury(task2[1]), 'task2 is buried') - test:ok(tube:bury(task3[1]), 'task3 is buried') - test:ok(tube:bury(task4[1]), 'task4 is buried') - - fiber.create(function() - for i = 1, 3 do - tube:take() - end - end) - - fiber.sleep(0.1) - test:is(tube:kick(3), 3, '3 tasks were kicken') - fiber.sleep(0.1) - - test:is(tube:peek(task1[1])[2], state.TAKEN, 'task1 unburied and taken') - test:is(tube:peek(task2[1])[2], state.TAKEN, 'task2 unburied and taken') - test:is(tube:peek(task3[1])[2], state.TAKEN, 'task3 unburied and taken') - test:is(tube:peek(task4[1])[2], state.BURIED, 'task4 is still buried') - - test:is(tube:kick(100500), 1, 'one task was unburied') - test:is(tube:peek(task4[1])[2], state.READY, 'task4 unburied and ready') - test:is(tube:delete(task4[1])[2], state.DONE, 'task4 was removed') - - local s, e = pcall(function() tube:peek(task4[1]) end) - test:ok(not s, "Task not found exception") - test:ok(string.match(e, "Task %d+ not found") ~= nil, - "Task not found message") - - end) -end) - -test:test('creating existing tube', function(test) - test:plan(2) - local s, e = pcall(function() queue.create_tube('test', 'fifo', { engine = engine }) end) - test:ok(not s, 'exception was thrown') - test:ok(e:match("Space 'test' already exists") ~= nil, 'text of exception') -end) - -test:test('tempspace', function(test) - if engine ~= 'vinyl' then - test:plan(2) - tube = queue.create_tube('test1', 'fifo', { temporary = true }) - test:ok(tube, 'tube was created') - local space_r = box.space._space:get{queue.tube.test1.raw.space.id} - test:ok(space_r[6].temporary, 'really tempspace') - else - test:plan(0) - end -end) - -test:test('disconnect test', function(test) - test:plan(6) - - tube:put(1) - tube:put(2) - tube:put(3) - - local task1 = tube:take(.1) - test:ok(task1, 'task1 was taken') - local task2 = tube:take(.1) - test:ok(task2, 'task2 was taken') - local task3 = tube:take(.1) - test:ok(task3, 'task3 was taken') - - queue._on_consumer_disconnect() - - test:is(tube:peek(task1[1])[2], state.READY, 'task1 was marked as READY') - test:is(tube:peek(task2[1])[2], state.READY, 'task2 was marked as READY') - test:is(tube:peek(task3[1])[2], state.READY, 'task3 was marked as READY') -end) - -test:test('if not exists tests', function(test) - if engine ~= 'vinyl' then - test:plan(1) - local tube_dup = queue.create_tube('test1', 'fifo', { if_not_exists = true }) - test:is(tube_dup, tube, '') - else - test:plan(0) - end -end) - -test:test('truncate test', function(test) - test:plan(3) - local len = tube.raw.space:count() - test:ok(len > 0, 'we have something in tube') - test:ok(len, tube:truncate(), 'we delete everything from tube') - test:is(tube.raw.space:count(), 0, 'nothing in tube after it') -end) - -test:test('if_not_exists test', function(test) - test:plan(2) - local tube = queue.create_tube('test_ine', 'fifo', { - if_not_exists = true, engine = engine - }) - local tube_new = queue.create_tube('test_ine', 'fifo', { - if_not_exists = true, engine = engine - }) - test:is(tube, tube_new, "if_not_exists if tube exists") - - queue.tube['test_ine'] = nil - local tube_new = queue.create_tube('test_ine', 'fifo', { - if_not_exists = true, engine = engine - }) - test:isnt(tube, tube_new, "if_not_exists if tube doesn't exists") -end) - -test:test('Get tasks by state test', function(test) - test:plan(2) - local tube = queue.create_tube('test_task_it', 'fifo') - - for i = 1, 10 do - tube:put('test_data' .. tostring(i)) - end - for i = 1, 4 do - tube:take(0.001) - end - - local count_taken = 0 - local count_ready = 0 - - for _, task in tube.raw:tasks_by_state(state.READY) do - if task[2] == state.READY then - count_ready = count_ready + 1 - end - end - - for _, task in tube.raw:tasks_by_state(state.TAKEN) do - if task[2] == state.TAKEN then - count_taken = count_taken + 1 - end - end - - test:is(count_ready, 6, 'Check tasks count in a ready state') - test:is(count_taken, 4, 'Check tasks count in a taken state') -end) - -tnt.finish() -os.exit(test:check() and 0 or 1) --- vim: set ft=lua : diff --git a/t/020-fifottl.t b/t/020-fifottl.t deleted file mode 100755 index 76942cc5..00000000 --- a/t/020-fifottl.t +++ /dev/null @@ -1,306 +0,0 @@ -#!/usr/bin/env tarantool -local fiber = require('fiber') - -local test = require('tap').test() -test:plan(16) - -local queue = require('queue') -local state = require('queue.abstract.state') -local queue_state = require('queue.abstract.queue_state') - -local qc = require('queue.compat') - -local tnt = require('t.tnt') -tnt.cfg{} - -local engine = os.getenv('ENGINE') or 'vinyl' - -test:ok(rawget(box, 'space'), 'box started') -test:ok(queue, 'queue is loaded') - -local tube = queue.create_tube('test', 'fifottl', { engine = engine }) -local tube2 = queue.create_tube('test_stat', 'fifottl', { engine = engine }) -test:ok(tube, 'test tube created') -test:is(tube.name, 'test', 'tube.name') -test:is(tube.type, 'fifottl', 'tube.type') - -test:test('Fifottl statistics', function(test) - test:plan(13) - tube2:put('stat_0') - tube2:put('stat_1') - tube2:put('stat_2') - tube2:put('stat_3') - tube2:put('stat_4') - tube2:put('stat_5', {delay=1000}) - tube2:delete(4) - tube2:take(.001) - tube2:release(0) - tube2:take(.001) - tube2:ack(0) - tube2:bury(1) - tube2:bury(2) - tube2:kick(1) - tube2:take(.001) - - local stats = queue.statistics('test_stat') - - -- check tasks statistics - test:is(stats.tasks.taken, 1, 'tasks.taken') - test:is(stats.tasks.buried, 1, 'tasks.buried') - test:is(stats.tasks.ready, 1, 'tasks.ready') - test:is(stats.tasks.done, 2, 'tasks.done') - test:is(stats.tasks.delayed, 1, 'tasks.delayed') - test:is(stats.tasks.total, 4, 'tasks.total') - - -- check function call statistics - test:is(stats.calls.delete, 1, 'calls.delete') - test:is(stats.calls.ack, 1, 'calls.ack') - test:is(stats.calls.take, 3, 'calls.take') - test:is(stats.calls.kick, 1, 'calls.kick') - test:is(stats.calls.bury, 2, 'calls.bury') - test:is(stats.calls.put, 6, 'calls.put') - test:is(stats.calls.release, 1, 'calls.release') -end) - - -test:test('put/take/peek', function(test) - test:plan(11) - - local task = tube:put('abc') - - test:ok(task, "task was put") - test:is(task[2], state.READY, "task.state") - - local peek = tube:peek(task[1]) - test:is_deeply(task[1], peek[1], "put and peek tasks are the same") - test:is_deeply(task[2], peek[2], "put and peek tasks are the same") - test:is_deeply(task[3], peek[3], "put and peek tasks are the same") - - local taken = tube:take( .1 ) - test:ok(taken, 'task was taken') - - test:is(task[1], taken[1], 'task.id') - test:is(taken[2], state.TAKEN, 'task.status') - - local ack = tube:ack(taken[1]) - test:ok(ack, 'task was acked') - - local s, e = pcall(function() tube:peek(task[1]) end) - test:ok(not s, "peek status") - test:ok(string.match(e, 'Task %d+ not found') ~= nil, 'peek error message') -end) - -test:test('delayed tasks', function(test) - test:plan(12) - - local task = tube:put('cde', { delay = 0.1, ttl = 0.1, ttr = 0.01 }) - test:ok(task, 'delayed task was put') - test:is(task[3], 'cde', 'task.data') - test:is(task[2], state.DELAYED, 'state is DELAYED') - - test:isnil(tube:take(.01), 'delayed task was not taken') - - local taken = tube:take(.15) - test:ok(taken, 'delayed task was taken after timeout') - test:is(taken[3], 'cde', 'task.data') - - local retaken = tube:take(0.05) - test:ok(retaken, "retake task after ttr") - test:is(retaken[3], 'cde', 'task.data') - - fiber.sleep(0.2) - local s, e = pcall(function() tube:peek(retaken[1]) end) - test:ok(not s, 'Task is not in database (TTL)') - test:ok(string.match(e, 'Task %d+ not found') ~= nil, 'peek error message') - - - s, e = pcall(function() tube:ack(retaken[1]) end) - test:ok(not s, 'Task is not ackable (TTL)') - test:ok(string.match(e, 'Task was not taken') ~= nil, 'peek error message') -end) - -test:test('delete/peek', function(test) - test:plan(10) - - local task = tube:put('abc') - test:ok(task, 'task was put') - test:is(task[2], state.READY, 'task is READY') - - local taken = tube:take(.1) - test:ok(taken, 'task was taken') - test:is(taken[3], 'abc', 'task.data') - test:is(taken[2], state.TAKEN, 'task is really taken') - - local removed = tube:delete(task[1]) - test:ok(removed, 'tube:delete') - test:is(removed[3], 'abc', 'removed.data') - test:is(removed[2], state.DONE, 'removed.status') - - local s, e = pcall(function() tube:ack(task[1]) end) - test:ok(not s, "Can't ack removed task") - test:ok(string.match(e, 'Task was not taken') ~= nil, 'peek error message') -end) - -test:test('bury/peek/kick', function(test) - test:plan(17) - - local task = tube:put('abc') - test:ok(task, 'task was put') - test:is(task[2], state.READY, 'task is READY') - - local taken = tube:take(.1) - test:ok(taken, 'task was taken') - test:is(taken[3], 'abc', 'task.data') - test:is(taken[2], state.TAKEN, 'task is really taken') - - local buried = tube:bury(task[1]) - test:ok(buried, 'tube:bury') - test:is(buried[3], 'abc', 'buried.data') - test:is(buried[2], state.BURIED, 'buried.status') - - local s, e = pcall(function() tube:ack(task[1]) end) - test:ok(not s, "Can't ack removed task") - test:ok(string.match(e, 'Task was not taken') ~= nil, 'peek error message') - - local peek = tube:peek(task[1]) - test:is(peek[1], buried[1], 'task was peek') - test:is(peek[2], buried[2], 'task.status') - test:is(peek[3], buried[3], 'task.data') - - fiber.create(function() - local retaken = tube:take(0.1) - test:ok(retaken, 'buried task was retaken') - - test:is(retaken[1], buried[1], 'task.id') - test:is(retaken[2], state.TAKEN, 'task.status') - test:is(retaken[3], buried[3], 'task.data') - end) - - tube:kick(1) - fiber.sleep(0.1) -end) - -test:test('if_not_exists test', function(test) - test:plan(2) - local tube = queue.create_tube('test_ine', 'fifottl', { - if_not_exists = true, engine = engine - }) - local tube_new = queue.create_tube('test_ine', 'fifottl', { - if_not_exists = true, engine = engine - }) - test:is(tube, tube_new, "if_not_exists if tube exists") - - queue.tube['test_ine'] = nil - local tube_new = queue.create_tube('test_ine', 'fifottl', { - if_not_exists = true, engine = engine - }) - test:isnt(tube, tube_new, "if_not_exists if tube doesn't exists") -end) - -test:test('touch test', function(test) - test:plan(3) - tube:put('abc', {ttl=0.2, ttr=0.1}) - local task = tube:take() - tube:touch(task[1], 0.3) - fiber.sleep(0.1) - test:is(task[2], 't') - task = tube:ack(task[1]) - test:is(task[2], '-') - test:isnt(task, nil) -end) - -test:test('read_only test', function(test) - test:plan(7) - tube:put('abc', { delay = 0.1 }) - local ttl_fiber = tube.raw.fiber - box.cfg{ read_only = true } - -- Wait for a change in the state of the queue to waiting no more than 10 seconds. - local rc = queue_state.poll(queue_state.states.WAITING, 10) - test:ok(rc, "queue state changed to waiting") - fiber.sleep(0.11) - test:is(ttl_fiber:status(), 'dead', - "check that background fiber is canceled") - test:isnil(tube.raw.fiber, - "check that background fiber object is cleaned") - if qc.check_version({1, 7}) then - local task = tube:take(0.2) - test:isnil(task, "check that task wasn't moved to ready state") - else - local stat, task = pcall(tube.take, tube, 0.2) - test:is(stat, false, "check that task wasn't taken") - end - box.cfg{ read_only = false } - local rc = queue_state.poll(queue_state.states.RUNNING, 10) - test:ok(rc, "queue state changed to running") - test:is(tube.raw.fiber:status(), 'suspended', - "check that background fiber started") - local task = tube:take() - test:isnt(task, nil, "check that we can take task") - tube:ack(task[1]) -end) - -test:test('ttl after delay test', function(test) - local TTL = 10 - local TTR = 20 - local DELTA = 5 - test:plan(2) - box.cfg{} - local tube = queue.create_tube('test_ttl_release', 'fifottl', { if_not_exists = true }) - tube:put({'test_task'}, { ttl = 10, ttr = 20 }) - tube:take() - tube:release(0, { delay = DELTA }) - local task = box.space.test_ttl_release:get(0) - test:is(task.ttl, (TTL + DELTA) * 1000000, 'check TTL after release') - test:is(task.ttr, TTR * 1000000, 'check TTR after release') -end) - --- gh-96: infinite loop after dropping a tube with a burried task -test:test('buried task in a dropped queue', function(test) - test:plan(1) - - local TASK_ID = 1 - local tube = queue.create_tube('test_drop_with_burried', 'fifottl', - {ttr = 0.1, if_not_exist = true}) - - tube:put({foo = 'bar'}) - local task = tube:take(0) - tube:bury(task[TASK_ID]) - - tube:drop() - fiber.sleep(0.2) - test:ok(true, 'queue does not hang') -end) - -test:test('Get tasks by state test', function(test) - test:plan(2) - local tube = queue.create_tube('test_task_it', 'fifottl') - - for i = 1, 10 do - tube:put('test_data' .. tostring(i)) - end - for i = 1, 4 do - tube:take(0.001) - end - - local count_taken = 0 - local count_ready = 0 - - for _, task in tube.raw:tasks_by_state(state.READY) do - if task[2] == state.READY then - count_ready = count_ready + 1 - end - end - - for _, task in tube.raw:tasks_by_state(state.TAKEN) do - if task[2] == state.TAKEN then - count_taken = count_taken + 1 - end - end - - test:is(count_ready, 6, 'Check tasks count in a ready state') - test:is(count_taken, 4, 'Check tasks count in a taken state') -end) - -tnt.finish() -os.exit(test:check() and 0 or 1) --- vim: set ft=lua : diff --git a/t/020-worker.t b/t/020-worker.t new file mode 100644 index 00000000..e0ff5822 --- /dev/null +++ b/t/020-worker.t @@ -0,0 +1,141 @@ +#!/usr/bin/perl + +use warnings; +use strict; +use utf8; +use open qw(:std :utf8); +use lib qw(lib ../lib); + +use Test::More; +use constant PLAN => 30; + +BEGIN { + system 'which tarantool_box >/dev/null 2>&1'; + if ($? == 0) { + plan tests => PLAN; + } else { + plan skip_all => 'tarantool_box not found'; + } +} + +use Encode qw(decode encode); +use Cwd 'cwd'; +use File::Spec::Functions 'catfile'; +#use feature 'state'; + + + +BEGIN { + # Подготовка объекта тестирования для работы с utf8 + my $builder = Test::More->builder; + binmode $builder->output, ":utf8"; + binmode $builder->failure_output, ":utf8"; + binmode $builder->todo_output, ":utf8"; + + use_ok 'DR::TarantoolQueue'; + use_ok 'Coro'; + use_ok 'DR::Tarantool', ':all'; + use_ok 'DR::Tarantool::StartTest'; + use_ok 'Time::HiRes', 'time'; + use_ok 'Coro::AnyEvent'; + use_ok 'DR::TarantoolQueue::Worker'; +} +my $t = DR::Tarantool::StartTest->run( + cfg => catfile(cwd, 'tarantool.cfg'), + script_dir => catfile(cwd) +); + + +my $q = DR::TarantoolQueue->new( + host => '127.0.0.1', + port => $t->primary_port, + space => 0, + tube => 'test_queue', +); + +isa_ok $q => 'DR::TarantoolQueue'; + +my (@f, @fh); +push @f => async { + ok $q->tnt->ping, 'tnt ping'; + push @fh => fileno($q->tnt->_llc->{fh}) +} for 1 .. 5; +$_->join for @f; +@f = (); +ok 5 == grep({ $_ == $fh[0] } @fh), 'connection established once'; + + +my $wrk = DR::TarantoolQueue::Worker->new( + queue => $q, + timeout => .5, + count => 10 +); + +async { + $wrk->run(sub {}); +}; + +my @tasks; +for (1 .. 31) { + push @tasks => $q->put(data => $_); +} + +Coro::AnyEvent::sleep 0.2; + +for (@tasks) { + $_ = eval { $q->peek(id => $_->id) }; +} +is scalar grep({ !defined($_) } @tasks), scalar @tasks, 'All tasks were ack'; + +is $wrk->stop, 0, 'workers were stopped'; +@tasks = (); + +async { + $wrk->run(sub { $_[0]->release(delay => 1) }); +}; +@tasks = (); + +for (1 .. 5) { + push @tasks => $q->put(data => {task => $_}); +} + +Coro::AnyEvent::sleep .2; +for (@tasks) { is $_->peek->status, 'delayed', 'task was released' }; + +is $wrk->stop, 0, 'workers were stopped'; + +is $q->statistics->{'space0.test_queue.tasks.delayed'}, 5, + '5 tasks are delayed'; +is $q->statistics->{'space0.test_queue.tasks.buried'}, 0, + '0 tasks are buried'; + +Coro::AnyEvent::sleep 1.1; + +is $q->statistics->{'space0.test_queue.tasks.delayed'}, 0, + '0 tasks are delayed'; +is $q->statistics->{'space0.test_queue.tasks.ready'}, 5, + '5 tasks are ready'; +async { + $wrk->run(sub { die 123 }); +}; +Coro::AnyEvent::sleep .2; +is $q->statistics->{'space0.test_queue.tasks.delayed'}, 0, + '0 tasks are delayed'; +is $q->statistics->{'space0.test_queue.tasks.buried'}, 5, + '5 tasks are buried'; +$wrk->stop; + +my $restarted = 0; +$wrk->restart(sub { $restarted++ }); +$wrk->restart_limit(15); +async {$wrk->run(sub {})}; +$q->put(data => $_) for 1 .. 99; +$wrk->stop; + +cmp_ok $restarted, '<=', 99 / $wrk->restart_limit, 'count of restarts'; +cmp_ok $restarted, '>=', 99 / $wrk->restart_limit / 2, 'count of restarts'; + + +END { + note $t->log if $ENV{DEBUG}; +} diff --git a/t/030-parallel.t b/t/030-parallel.t new file mode 100644 index 00000000..8753ed1a --- /dev/null +++ b/t/030-parallel.t @@ -0,0 +1,100 @@ +#!/usr/bin/perl + +use warnings; +use strict; +use utf8; +use open qw(:std :utf8); +use lib qw(lib ../lib); + +use Test::More; +use constant PLAN => 18; + +BEGIN { + system 'which tarantool_box >/dev/null 2>&1'; + if ($? == 0) { + plan tests => PLAN; + } else { + plan skip_all => 'tarantool_box not found'; + } +} + +use Encode qw(decode encode); +use Cwd 'cwd'; +use File::Spec::Functions 'catfile'; +#use feature 'state'; + + + +BEGIN { + # Подготовка объекта тестирования для работы с utf8 + my $builder = Test::More->builder; + binmode $builder->output, ":utf8"; + binmode $builder->failure_output, ":utf8"; + binmode $builder->todo_output, ":utf8"; + + use_ok 'DR::TarantoolQueue'; + use_ok 'Coro'; + use_ok 'DR::Tarantool', ':all'; + use_ok 'DR::Tarantool::StartTest'; + use_ok 'Time::HiRes', 'time'; + use_ok 'Coro::AnyEvent'; + use_ok 'DR::TarantoolQueue::Worker'; +} +my $t = DR::Tarantool::StartTest->run( + cfg => catfile(cwd, 'tarantool.cfg'), + script_dir => catfile(cwd) +); + + +my $q = DR::TarantoolQueue->new( + host => '127.0.0.1', + port => $t->primary_port, + space => 0, + tube => 'test_queue', +); + +isa_ok $q => 'DR::TarantoolQueue'; + +my (@f, @fh); +push @f => async { + ok $q->tnt->ping, 'tnt ping'; + push @fh => fileno($q->tnt->_llc->{fh}) +} for 1 .. 5; +$_->join for @f; +@f = (); +ok 5 == grep({ $_ == $fh[0] } @fh), 'connection established once'; + + +my $wrk = DR::TarantoolQueue::Worker->new( + queue => $q, + timeout => 1, + count => 100 +); + +async { + $wrk->run(sub { Coro::AnyEvent::sleep 1; die 123 }); +}; + + +my @tasks; +for (1 .. 100) { + push @tasks => $q->put(tube => 'test_queue'); +} + +my @status; +push @status => $_->peek->status for @tasks; + +cmp_ok 0, '<=', scalar grep({ $_ eq 'ready' } @status), 'some tasks are ready'; +cmp_ok 0, '<', scalar grep({ $_ eq 'taken' } @status), 'some tasks are taken'; +Coro::AnyEvent::sleep 0.5; + +@status = (); +push @status => $_->peek->status for @tasks; + +is scalar grep({ $_ eq 'ready' } @status), 0, 'no tasks are ready'; +Coro::AnyEvent::sleep 1; + +@status = (); +push @status => $_->peek->status for @tasks; +is scalar grep({ $_ eq 'buried' } @status), 100, 'all tasks are buried'; + diff --git a/t/030-utube.t b/t/030-utube.t deleted file mode 100755 index a5b5057d..00000000 --- a/t/030-utube.t +++ /dev/null @@ -1,248 +0,0 @@ -#!/usr/bin/env tarantool -local yaml = require('yaml') -local fiber = require('fiber') - -local test = (require('tap')).test() -test:plan(12) - -local queue = require('queue') -local state = require('queue.abstract.state') - -local tnt = require('t.tnt') -tnt.cfg{} - -local engine = os.getenv('ENGINE') or 'memtx' - -test:ok(rawget(box, 'space'), 'box started') -test:ok(queue, 'queue is loaded') - -local tube = queue.create_tube('test', 'utube', { engine = engine }) -local tube2 = queue.create_tube('test_stat', 'utube', { engine = engine }) -local tube_ready, tube2_ready -if engine ~= 'vinyl' then - tube_ready = queue.create_tube('test_ready', 'utube', - { engine = engine, storage_mode = queue.driver.utube.STORAGE_MODE_READY_BUFFER }) - tube2_ready = queue.create_tube('test_stat_ready', 'utube', - { engine = engine, storage_mode = queue.driver.utube.STORAGE_MODE_READY_BUFFER }) -end -test:ok(tube, 'test tube created') -test:is(tube.name, 'test', 'tube.name') -test:is(tube.type, 'utube', 'tube.type') - -test:test('Utube statistics', function(test) - if engine ~= 'vinyl' then - test:plan(13 * 2) - else - test:plan(13) - end - for _, tube_stat in ipairs({tube2, tube2_ready}) do - if tube_stat == nil then - break - end - - tube_stat:put('stat_0') - tube_stat:put('stat_1') - tube_stat:put('stat_2') - tube_stat:put('stat_3') - tube_stat:put('stat_4') - tube_stat:delete(4) - tube_stat:take(.001) - tube_stat:release(0) - tube_stat:take(.001) - tube_stat:ack(0) - tube_stat:bury(1) - tube_stat:bury(2) - tube_stat:kick(1) - tube_stat:take(.001) - - local stats = queue.statistics(tube_stat.name) - - -- check tasks statistics - test:is(stats.tasks.taken, 1, 'tasks.taken') - test:is(stats.tasks.buried, 1, 'tasks.buried') - test:is(stats.tasks.ready, 1, 'tasks.ready') - test:is(stats.tasks.done, 2, 'tasks.done') - test:is(stats.tasks.delayed, 0, 'tasks.delayed') - test:is(stats.tasks.total, 3, 'tasks.total') - - -- check function call statistics - test:is(stats.calls.delete, 1, 'calls.delete') - test:is(stats.calls.ack, 1, 'calls.ack') - test:is(stats.calls.take, 3, 'calls.take') - test:is(stats.calls.kick, 1, 'calls.kick') - test:is(stats.calls.bury, 2, 'calls.bury') - test:is(stats.calls.put, 5, 'calls.put') - test:is(stats.calls.release, 1, 'calls.release') - end -end) - - -test:test('Easy put/take/ack', function(test) - if engine ~= 'vinyl' then - test:plan(12 * 2) - else - test:plan(12) - end - - for _, test_tube in ipairs({tube, tube_ready}) do - if test_tube == nil then - break - end - - test:ok(test_tube:put(123, {utube = 1}), 'task was put') - test:ok(test_tube:put(345, {utube = 1}), 'task was put') - local task = test_tube:take() - test:ok(task, 'task was taken') - test:is(task[2], state.TAKEN, 'task status') - test:is(task[3], 123, 'task.data') - test:ok(test_tube:take(.1) == nil, 'second task was not taken (the same tube)') - - task = test_tube:ack(task[1]) - test:ok(task, 'task was acked') - test:is(task[2], '-', 'task status') - test:is(task[3], 123, 'task.data') - - task = test_tube:take(.1) - test:ok(task, 'task2 was taken') - test:is(task[3], 345, 'task.data') - test:is(task[2], state.TAKEN, 'task.status') - end -end) - -test:test('ack in utube', function(test) - if engine ~= 'vinyl' then - test:plan(8 * 2) - else - test:plan(8) - end - - for _, test_tube in ipairs({tube, tube_ready}) do - if test_tube == nil then - break - end - - test:ok(test_tube:put(123, {utube = 'abc'}), 'task was put') - test:ok(test_tube:put(345, {utube = 'abc'}), 'task was put') - - local state = 0 - fiber.create(function() - fiber.sleep(0.1) - local taken = test_tube:take() - test:ok(taken, 'second task was taken') - test:is(taken[3], 345, 'task.data') - state = state + 1 - end) - - local taken = test_tube:take(.1) - state = 1 - test:ok(taken, 'task was taken') - test:is(taken[3], 123, 'task.data') - fiber.sleep(0.3) - test:is(state, 1, 'state was not changed') - test_tube:ack(taken[1]) - fiber.sleep(0.2) - test:is(state, 2, 'state was changed') - end -end) -test:test('bury in utube', function(test) - if engine ~= 'vinyl' then - test:plan(8 * 2) - else - test:plan(8) - end - - for _, test_tube in ipairs({tube, tube_ready}) do - if test_tube == nil then - break - end - - test:ok(test_tube:put(567, {utube = 'cde'}), 'task was put') - test:ok(test_tube:put(789, {utube = 'cde'}), 'task was put') - - local state = 0 - fiber.create(function() - fiber.sleep(0.1) - local taken = test_tube:take() - test:ok(taken, 'second task was taken') - test:is(taken[3], 789, 'task.data') - state = state + 1 - end) - - local taken = test_tube:take(.1) - state = 1 - test:ok(taken, 'task was taken') - test:is(taken[3], 567, 'task.data') - fiber.sleep(0.3) - test:is(state, 1, 'state was not changed') - test_tube:bury(taken[1]) - fiber.sleep(0.2) - test:is(state, 2, 'state was changed') - end -end) -test:test('instant bury', function(test) - if engine ~= 'vinyl' then - test:plan(1 * 2) - else - test:plan(1) - end - tube:put(1, {ttr=60}) - local taken = tube:take(.1) - test:is(tube:bury(taken[1])[2], '!', 'task is buried') - - if tube_ready ~= nil then - tube_ready:put(1, {ttr=60}) - local taken = tube_ready:take(.1) - test:is(tube_ready:bury(taken[1])[2], '!', 'task is buried') - end -end) - -test:test('if_not_exists test', function(test) - test:plan(2) - local tube = queue.create_tube('test_ine', 'utube', { - if_not_exists = true, engine = engine - }) - local tube_new = queue.create_tube('test_ine', 'utube', { - if_not_exists = true, engine = engine - }) - test:is(tube, tube_new, "if_not_exists if tube exists") - - queue.tube['test_ine'] = nil - local tube_new = queue.create_tube('test_ine', 'utube', { - if_not_exists = true, engine = engine - }) - test:isnt(tube, tube_new, "if_not_exists if tube doesn't exists") -end) - -test:test('Get tasks by state test', function(test) - test:plan(2) - local tube = queue.create_tube('test_task_it', 'utube') - - for i = 1, 10 do - tube:put('test_data' .. tostring(i), { utube = i }) - end - for i = 1, 4 do - tube:take(0.001) - end - - local count_taken = 0 - local count_ready = 0 - - for _, task in tube.raw:tasks_by_state(state.READY) do - if task[2] == state.READY then - count_ready = count_ready + 1 - end - end - - for _, task in tube.raw:tasks_by_state(state.TAKEN) do - if task[2] == state.TAKEN then - count_taken = count_taken + 1 - end - end - - test:is(count_ready, 6, 'Check tasks count in a ready state') - test:is(count_taken, 4, 'Check tasks count in a taken state') -end) - -tnt.finish() -os.exit(test:check() and 0 or 1) --- vim: set ft=lua : diff --git a/t/040-utubettl.t b/t/040-utubettl.t deleted file mode 100755 index f6f811e9..00000000 --- a/t/040-utubettl.t +++ /dev/null @@ -1,429 +0,0 @@ -#!/usr/bin/env tarantool -local yaml = require('yaml') -local fiber = require('fiber') - -local test = (require('tap')).test() -test:plan(18) - -local queue = require('queue') -local state = require('queue.abstract.state') -local queue_state = require('queue.abstract.queue_state') - -local qc = require('queue.compat') - -local tnt = require('t.tnt') -tnt.cfg{} - -local engine = os.getenv('ENGINE') or 'memtx' - -test:ok(rawget(box, 'space'), 'box started') -test:ok(queue, 'queue is loaded') - -local tube = queue.create_tube('test', 'utubettl', { engine = engine }) -local tube2 = queue.create_tube('test_stat', 'utubettl', { engine = engine }) -local tube_ready, tube2_ready -if engine ~= 'vinyl' then - tube_ready = queue.create_tube('test_ready', 'utubettl', - { engine = engine, storage_mode = queue.driver.utubettl.STORAGE_MODE_READY_BUFFER }) - tube2_ready = queue.create_tube('test_stat_ready', 'utubettl', - { engine = engine, storage_mode = queue.driver.utubettl.STORAGE_MODE_READY_BUFFER }) -end -test:ok(tube, 'test tube created') -test:is(tube.name, 'test', 'tube.name') -test:is(tube.type, 'utubettl', 'tube.type') - -test:test('Utubettl statistics', function(test) - if engine ~= 'vinyl' then - test:plan(13 * 2) - else - test:plan(13) - end - for _, tube_stat in ipairs({tube2, tube2_ready}) do - if tube_stat == nil then - break - end - - tube_stat:put('stat_0') - tube_stat:put('stat_1') - tube_stat:put('stat_2') - tube_stat:put('stat_3') - tube_stat:put('stat_4') - tube_stat:put('stat_5', {delay=1000}) - tube_stat:delete(4) - tube_stat:take(.001) - tube_stat:release(0) - tube_stat:take(.001) - tube_stat:ack(0) - tube_stat:bury(1) - tube_stat:bury(2) - tube_stat:kick(1) - tube_stat:take(.001) - - local stats = queue.statistics(tube_stat.name) - - -- check tasks statistics - test:is(stats.tasks.taken, 1, 'tasks.taken') - test:is(stats.tasks.buried, 1, 'tasks.buried') - test:is(stats.tasks.ready, 1, 'tasks.ready') - test:is(stats.tasks.done, 2, 'tasks.done') - test:is(stats.tasks.delayed, 1, 'tasks.delayed') - test:is(stats.tasks.total, 4, 'tasks.total') - - -- check function call statistics - test:is(stats.calls.delete, 1, 'calls.delete') - test:is(stats.calls.ack, 1, 'calls.ack') - test:is(stats.calls.take, 3, 'calls.take') - test:is(stats.calls.kick, 1, 'calls.kick') - test:is(stats.calls.bury, 2, 'calls.bury') - test:is(stats.calls.put, 6, 'calls.put') - test:is(stats.calls.release, 1, 'calls.release') - end -end) - - -test:test('Easy put/take/ack', function(test) - if engine ~= 'vinyl' then - test:plan(12 * 2) - else - test:plan(12) - end - - for _, test_tube in ipairs({tube, tube_ready}) do - if test_tube == nil then - break - end - - test:ok(test_tube:put(123, {utube = 1}), 'task was put') - test:ok(test_tube:put(345, {utube = 1}), 'task was put') - local task = test_tube:take() - test:ok(task, 'task was taken') - test:is(task[2], state.TAKEN, 'task status') - test:is(task[3], 123, 'task.data') - test:ok(test_tube:take(.1) == nil, 'second task was not taken (the same tube)') - - task = test_tube:ack(task[1]) - test:ok(task, 'task was acked') - test:is(task[2], '-', 'task status') - test:is(task[3], 123, 'task.data') - - task = test_tube:take(.1) - test:ok(task, 'task2 was taken') - test:is(task[3], 345, 'task.data') - test:is(task[2], state.TAKEN, 'task.status') - end -end) - -test:test('ttr put/take', function(test) - if engine ~= 'vinyl' then - test:plan(3 * 2) - else - test:plan(3) - end - - local my_queue = queue.create_tube('trr_test', 'utubettl', { engine = engine }) - test:ok(my_queue:put('ttr1', { ttr = 1 }), 'put ttr task') - test:ok(my_queue:take(0.1) ~= nil, 'take this task') - fiber.sleep(1.1) - local task = my_queue:peek(0) - test:is(task[2], state.READY, 'Ready state returned after one second') - - if engine ~= 'vinyl' then - local my_queue_ready = queue.create_tube('trr_test_v2', 'utubettl', - { engine = engine, storage_mode = queue.driver.utubettl.STORAGE_MODE_READY_BUFFER }) - test:ok(my_queue_ready:put('ttr1', { ttr = 1 }), 'put ttr task') - test:ok(my_queue_ready:take(0.1) ~= nil, 'take this task') - fiber.sleep(1.1) - local task = my_queue_ready:peek(0) - test:is(task[2], state.READY, 'Ready state returned after one second') - end -end) - -test:test('ack in utube', function(test) - if engine ~= 'vinyl' then - test:plan(8 * 2) - else - test:plan(8) - end - - for _, test_tube in ipairs({tube, tube_ready}) do - if test_tube == nil then - break - end - - test:ok(test_tube:put(123, {utube = 'abc'}), 'task was put') - test:ok(test_tube:put(345, {utube = 'abc'}), 'task was put') - - local state = 0 - fiber.create(function() - fiber.sleep(0.1) - local taken = test_tube:take() - test:ok(taken, 'second task was taken') - test:is(taken[3], 345, 'task.data') - state = state + 1 - end) - - local taken = test_tube:take(.1) - state = 1 - test:ok(taken, 'task was taken') - test:is(taken[3], 123, 'task.data') - fiber.sleep(0.3) - test:is(state, 1, 'state was not changed') - test_tube:ack(taken[1]) - fiber.sleep(0.2) - test:is(state, 2, 'state was changed') - end -end) -test:test('bury in utube', function(test) - if engine ~= 'vinyl' then - test:plan(8 * 2) - else - test:plan(8) - end - - for _, test_tube in ipairs({tube, tube_ready}) do - if test_tube == nil then - break - end - - test:ok(test_tube:put(567, {utube = 'cde'}), 'task was put') - test:ok(test_tube:put(789, {utube = 'cde'}), 'task was put') - - local state = 0 - fiber.create(function() - fiber.sleep(0.1) - local taken = test_tube:take() - test:ok(taken, 'second task was taken') - test:is(taken[3], 789, 'task.data') - state = state + 1 - end) - - local taken = test_tube:take(.1) - state = 1 - test:ok(taken, 'task was taken') - test:is(taken[3], 567, 'task.data') - fiber.sleep(0.3) - test:is(state, 1, 'state was not changed') - test_tube:bury(taken[1]) - fiber.sleep(0.2) - test:is(state, 2, 'state was changed') - end -end) - -test:test('instant bury', function(test) - if engine ~= 'vinyl' then - test:plan(1 * 2) - else - test:plan(1) - end - - tube:put(1, {ttr=60}) - local taken = tube:take(.1) - test:is(tube:bury(taken[1])[2], '!', 'task is buried') - - if tube_ready ~= nil then - tube_ready:put(1, {ttr=60}) - taken = tube_ready:take(.1) - test:is(tube_ready:bury(taken[1])[2], '!', 'task is buried') - end -end) - -test:test('priority in utube', function(test) - if engine ~= 'vinyl' then - test:plan(8 * 2) - else - test:plan(8) - end - - for _, test_tube in ipairs({tube, tube_ready}) do - if test_tube == nil then - break - end - - test:ok(test_tube:put(670, {utube = 'dee', pri = 1}), 'task was put') - test:ok(test_tube:put(671, {utube = 'dee', pri = 0}), 'task was put') - - local taken = test_tube:take(.1) - test:ok(taken, 'task was taken ' .. taken[1]) - test:is(taken[3], 671, 'task.data') - - test_tube:release(taken[1]) - - taken = test_tube:take(.1) - test:ok(taken, 'task was taken ' .. taken[1]) - test:is(taken[3], 671, 'task.data') - test_tube:ack(taken[1]) - - taken = test_tube:take(.1) - test:ok(taken, 'task was taken ' .. taken[1]) - test:is(taken[3], 670, 'task.data') - test_tube:ack(taken[1]) - end -end) - -test:test('release in utube', function(test) - if engine ~= 'vinyl' then - test:plan(8 * 2) - else - test:plan(8) - end - - for _, test_tube in ipairs({tube, tube_ready}) do - if test_tube == nil then - break - end - - test:ok(test_tube:put(678, {utube = 'def'}), 'task was put') - test:ok(test_tube:put(890, {utube = 'def'}), 'task was put') - - local state = 0 - fiber.create(function() - fiber.sleep(0.1) - local taken = test_tube:take() - test:ok(taken, 'first task was taken again') - test:is(taken[3], 678, 'task.data') - state = state + 1 - end) - - local taken = test_tube:take(.1) - state = 1 - test:ok(taken, 'task was taken ' .. taken[1]) - test:is(taken[3], 678, 'task.data') - fiber.sleep(0.3) - test:is(state, 1, 'state was not changed') - test_tube:release(taken[1]) - fiber.sleep(0.2) - test:is(state, 2, 'state was changed') - end -end) - -test:test('release[delay] in utube', function(test) - if engine ~= 'vinyl' then - test:plan(8 * 2) - else - test:plan(8) - end - - for _, test_tube in ipairs({tube, tube_ready}) do - if test_tube == nil then - break - end - - test:ok(test_tube:put(789, {utube = 'efg'}), 'task was put') - test:ok(test_tube:put(901, {utube = 'efg'}), 'task was put') - - local state = 0 - fiber.create(function() - fiber.sleep(0.1) - local taken = test_tube:take() - test:ok(taken, 'second task was taken') - test:is(taken[3], 901, 'task.data') - state = state + 1 - end) - - local taken = test_tube:take(.1) - state = 1 - test:ok(taken, 'task was taken ' .. taken[1]) - test:is(taken[3], 789, 'task.data') - fiber.sleep(0.3) - test:is(state, 1, 'state was not changed') - test_tube:release(taken[1], { delay = 10 }) - fiber.sleep(0.2) - test:is(state, 2, 'state was changed') - end -end) - -test:test('if_not_exists test', function(test) - test:plan(2) - local tube = queue.create_tube('test_ine', 'utubettl', { - if_not_exists = true, engine = engine - }) - local tube_new = queue.create_tube('test_ine', 'utubettl', { - if_not_exists = true, engine = engine - }) - test:is(tube, tube_new, "if_not_exists if tube exists") - - queue.tube['test_ine'] = nil - local tube_new = queue.create_tube('test_ine', 'utubettl', { - if_not_exists = true, engine = engine - }) - test:isnt(tube, tube_new, "if_not_exists if tube doesn't exists") -end) - -test:test('read_only test', function(test) - test:plan(7) - tube:put('abc', { delay = 0.1 }) - local ttl_fiber = tube.raw.fiber - box.cfg{ read_only = true } - -- Wait for a change in the state of the queue to waiting no more than 10 seconds. - local rc = queue_state.poll(queue_state.states.WAITING, 10) - test:ok(rc, "queue state changed to waiting") - fiber.sleep(0.11) - test:is(ttl_fiber:status(), 'dead', - "check that background fiber is canceled") - test:isnil(tube.raw.fiber, - "check that background fiber object is cleaned") - if qc.check_version({1, 7}) then - local task = tube:take(0.2) - test:isnil(task, "check that task wasn't moved to ready state") - else - local stat, task = pcall(tube.take, tube, 0.2) - test:is(stat, false, "check that task wasn't taken") - end - box.cfg{ read_only = false } - local rc = queue_state.poll(queue_state.states.RUNNING, 10) - test:ok(rc, "queue state changed to running") - test:is(tube.raw.fiber:status(), 'suspended', - "check that background fiber started") - local task = tube:take() - test:isnt(task, nil, "check that we can take task") - tube:ack(task[1]) -end) - -test:test('ttl after delay test', function(test) - local TTL = 10 - local TTR = 20 - local DELTA = 5 - test:plan(2) - box.cfg{} - local tube = queue.create_tube('test_ttl_release', 'utubettl', { if_not_exists = true }) - tube:put({'test_task'}, { ttl = 10, ttr = 20 }) - tube:take() - tube:release(0, { delay = DELTA }) - local task = box.space.test_ttl_release:get(0) - test:is(task.ttl, (TTL + DELTA) * 1000000, 'check TTL after release') - test:is(task.ttr, TTR * 1000000, 'check TTR after release') -end) - -test:test('Get tasks by state test', function(test) - test:plan(2) - local tube = queue.create_tube('test_task_it', 'utubettl') - - for i = 1, 10 do - tube:put('test_data' .. tostring(i), { utube = i }) - end - for i = 1, 4 do - tube:take(0.001) - end - - local count_taken = 0 - local count_ready = 0 - - for _, task in tube.raw:tasks_by_state(state.READY) do - if task[2] == state.READY then - count_ready = count_ready + 1 - end - end - - for _, task in tube.raw:tasks_by_state(state.TAKEN) do - if task[2] == state.TAKEN then - count_taken = count_taken + 1 - end - end - - test:is(count_ready, 6, 'Check tasks count in a ready state') - test:is(count_taken, 4, 'Check tasks count in a taken state') -end) - -tnt.finish() -os.exit(test:check() and 0 or 1) --- vim: set ft=lua : diff --git a/t/050-ttl.t b/t/050-ttl.t deleted file mode 100755 index 8f26b4fb..00000000 --- a/t/050-ttl.t +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env tarantool -local fiber = require('fiber') - -local test = require('tap').test() -test:plan(7) - -local queue = require('queue') - -local engine = os.getenv('ENGINE') or 'memtx' - -local tnt = require('t.tnt') -tnt.cfg{ - wal_mode = (engine == 'memtx' and 'none' or nil) -} - -local ttl = 0.1 - -test:ok(queue, 'queue is loaded') - -local function test_take_after_ttl(test, tube, ttl) - local attempts, max_attempts = 0, 2 - local before = fiber.time64() - local after = before - - while after - before < ttl * 1000000 and attempts < max_attempts do - attempts = attempts + 1 - fiber.sleep(ttl) - after = fiber.time64() - end - - if after - before < ttl * 1000000 then - test:fail('failed to sleep ttl, is system clock changed?') - else - test:isnil(tube:take(.1), 'no task is taken') - end -end - -test:test('one message per queue ffttl', function (test) - test:plan(20) - local tube = queue.create_tube('ompq_ffttl', 'fifottl', { engine = engine }) - for i = 1, 20 do - tube:put('ompq_' .. i, {ttl=ttl}) - - test_take_after_ttl(test, tube, ttl) - end -end) - -test:test('one message per queue utttl', function (test) - test:plan(20) - local tube = queue.create_tube('ompq_utttl', 'utubettl', { engine = engine }) - for i = 1, 20 do - tube:put('ompq_' .. i, {ttl=ttl}) - - test_take_after_ttl(test, tube, ttl) - end -end) - -test:test('one message per queue utttl_ready', function (test) - if engine == 'vinyl' then - return - end - - test:plan(20) - local tube = queue.create_tube('ompq_utttl_ready', 'utubettl', - { engine = engine, storage_mode = queue.driver.utubettl.STORAGE_MODE_READY_BUFFER }) - for i = 1, 20 do - tube:put('ompq_' .. i, {ttl=ttl}) - - test_take_after_ttl(test, tube, ttl) - end -end) - -test:test('many messages, one queue ffttl', function (test) - test:plan(20) - for i = 1, 20 do - local tube = queue.create_tube('mmpq_ffttl_' .. i, 'fifottl', { engine = engine }) - tube:put('mmpq_' .. i, {ttl=ttl}) - - test_take_after_ttl(test, tube, ttl) - end -end) - -test:test('many messages, one queue utttl', function (test) - test:plan(20) - for i = 1, 20 do - local tube = queue.create_tube('mmpq_utttl_' .. i, 'utubettl', { engine = engine }) - tube:put('mmpq_' .. i, {ttl=ttl}) - - test_take_after_ttl(test, tube, ttl) - end -end) - -test:test('many messages, one queue utttl_ready', function (test) - if engine == 'vinyl' then - return - end - - test:plan(20) - for i = 1, 20 do - local tube = queue.create_tube('mmpq_utttl_ready_' .. i, 'utubettl', - { engine = engine, storage_mode = queue.driver.utubettl.STORAGE_MODE_READY_BUFFER }) - tube:put('mmpq_' .. i, {ttl=ttl}) - - test_take_after_ttl(test, tube, ttl) - end -end) - -tnt.finish() -os.exit(test:check() and 0 or 1) --- vim: set ft=lua : diff --git a/t/060-async.t b/t/060-async.t deleted file mode 100755 index 8737501a..00000000 --- a/t/060-async.t +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env tarantool -local yaml = require('yaml') -local fiber = require('fiber') - -local test = require('tap').test() -test:plan(6) - -local queue = require('queue') -local state = require('queue.abstract.state') - -local tnt = require('t.tnt') -tnt.cfg{} - -local engine = os.getenv('ENGINE') or 'memtx' - -test:ok(rawget(box, 'space'), 'box started') -test:ok(queue, 'queue is loaded') - -local tube = queue.create_tube('test', 'fifo', { engine = engine }) -test:ok(tube, 'test tube created') -test:is(tube.name, 'test', 'tube.name') -test:is(tube.type, 'fifo', 'tube.type') - -test:test('concurent take', function(test) - test:plan(16) - - local channel = fiber.channel(1000) - test:ok(channel, 'channel created') - - local res = {} - for i = 1, 5 do - fiber.create(function(i) - local taken = tube:take(1) - test:ok(taken, 'Task was taken ' .. i) - table.insert(res, { taken }) - channel:put(true) - end, i) - end - - fiber.sleep(.1) - test:ok(tube:put(1), 'task 1 was put') - - for i = 2, 5 do - fiber.create(function(i) - fiber.sleep(.5 / i) - test:ok(tube:put(i), 'task ' .. i .. ' was put') - end, i) - end - - for i = 1, 5 do - test:ok(channel:get(1 / i), 'take was done ' .. i) - end -end) - - -tnt.finish() -os.exit(test:check() and 0 or 1) --- vim: set ft=lua: diff --git a/t/070-compat.t b/t/070-compat.t deleted file mode 100755 index a78b5dd0..00000000 --- a/t/070-compat.t +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env tarantool - -local test = require('tap').test() -test:plan(3) - -local qcompat = require('queue.compat') - -test:ok(qcompat, 'queue compatibility layer exists') - -test:test('*_version', function(test) - test:plan(10) - - local split_version = qcompat.split_version - local check_version = qcompat.check_version - - test:is_deeply(split_version("1.6.8-173"), {"1", "6", "8", "173"}, "check split_version 1") - test:is_deeply(split_version("1.7.1-0"), {"1", "7", "1", "0"}, "check split_version 2") - test:is_deeply(split_version("1.7.1"), {"1", "7", "1"}, "check split_version 3") - - test:is(check_version({1, 7, 1}, "1.8.1-0"), true, "check supported version") - test:is(check_version({1, 7, 1}, "1.7.1-0"), true, "check supported version") - test:is(check_version({1, 7, 1}, "1.7.1-1"), true, "check supported version") - test:is(check_version({1, 7, 2}, "1.8.1"), true, "check supported version") - test:is(check_version({1, 7, 1}, "1.6.9"), false, "check unsupported version") - test:is(check_version({1, 7, 1}, "1.6.9-100"), false, "check unsupported version") - test:is(check_version({1, 7, 1}, "1.6.9-100"), false, "check unsupported version") -end) - -test:test("check compatibility names", function(test) - test:plan(7) - - local vinyl_name = qcompat.vinyl_name - local num_name = qcompat.num_type - local str_name = qcompat.str_type - - test:is(vinyl_name("1.7.1-168"), "vinyl", "check new name (vinyl)") - test:is(num_name("1.7.2-1"), "unsigned", "check new name (unsigned)") - test:is(num_name("1.6.9-168"), "num", "check old name (num)") - test:is(num_name("1.7.1-168"), "num", "check old name (num)") - test:is(str_name("1.7.2-1"), "string", "check new name (string)") - test:is(str_name("1.6.9-168"), "str", "check old name (str)") - test:is(str_name("1.7.1-168"), "str", "check old name (str)") -end) - -os.exit(test:check() and 0 or 1) --- vim: set ft=lua: diff --git a/t/080-otc-cb.t b/t/080-otc-cb.t deleted file mode 100755 index 7a6b5d70..00000000 --- a/t/080-otc-cb.t +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env tarantool -local test = require('tap').test() -test:plan(1) - -local tnt = require('t.tnt') -tnt.cfg{} - -local engine = os.getenv('ENGINE') or 'memtx' - -local queue = require('queue') - -local function tube_check_simple(tube) - tube:put{123} - local task = tube:take(0) - tube:ack(task[1]) -end - -test:test('on_task_change callback', function(test) - test:plan(5) - local cnt = 0 - - -- simple callbacks, to check that they're being called - local function cb(t1, t2) cnt = cnt + 1 end - local function new_cb(t1, t2) cnt = cnt + 2 end - - -- init with initial callback - local tube = queue.create_tube('test2', 'fifo', { - on_task_change = cb, engine = engine - }) - - tube_check_simple(tube) - test:is(cnt, 3, 'check counter') - - -- set new callback - test:is(tube:on_task_change(new_cb), cb, "check rv of on_task_change") - - tube_check_simple(tube) - test:is(cnt, 9, 'check counter after new cb is applied') - - -- delete callback - test:is(tube:on_task_change(), new_cb, "check rv of on_task_change") - - tube_check_simple(tube) - test:is(cnt, 9, 'check counter after new cb is applied') -end) - -tnt.finish() -os.exit(test:check() and 0 or 1) --- vim: set ft=lua : diff --git a/t/090-grant-check.t b/t/090-grant-check.t deleted file mode 100755 index 03dde415..00000000 --- a/t/090-grant-check.t +++ /dev/null @@ -1,242 +0,0 @@ -#!/usr/bin/env tarantool -local test = require('tap').test() -test:plan(9) - -local test_user = 'test' -local test_pass = '1234' -local test_host = 'localhost' -local test_port = '3388' - -local uri = require('uri') -local netbox = require('net.box') - -local tnt = require('t.tnt') - -tnt.cfg{ - listen = ('%s:%s'):format(test_host, test_port) - -- listen = uri.format({ host = test_host, service = test_port }) -} - -local engine = os.getenv('ENGINE') or 'memtx' - -local qc = require('queue.compat') - -local test_drivers_grant_cases = { - { - name = 'fifo', - queue_type = 'fifo', - }, - { - name = 'fifottl', - queue_type = 'fifottl', - }, - { - name = 'utube_default', - queue_type = 'utube', - storage_mode = 'default', - }, - { - name = 'utube_ready_buffer', - queue_type = 'utube', - storage_mode = 'ready_buffer', - }, - { - name = 'utubettl_default', - queue_type = 'utubettl', - storage_mode = 'default', - }, - { - name = 'utubettl_ready_buffer', - queue_type = 'utubettl', - storage_mode = 'ready_buffer', - }, -} - -for _, tc in pairs(test_drivers_grant_cases) do - test:test('test dirvers grant ' .. tc.name, function(test) - local queue = require('queue') - box.schema.user.create(test_user, { password = test_pass }) - - test:plan(2) - - local tube_opts = { engine = engine } - if tc.storage_mode ~= nil and tc.storage_mode ~= 'default' then - tube_opts.storage_mode = tc.storage_mode - tube_opts.engine = 'memtx' - end - local tube = queue.create_tube('test', tc.queue_type, tube_opts) - tube:put('help') - - tube:grant(test_user) - - box.session.su(test_user) - local a = tube:take() - test:is(a[1], 0, 'we aren\'t getting any error') - - local c = tube:ack(a[1]) - test:is(c[1], 0, 'we aren\'t getting any error') - - box.session.su('admin') - - box.schema.user.drop(test_user) - tube:drop() - end) -end - -test:test('check for space grants', function(test) - -- prepare for tests - local queue = require('queue') - box.schema.user.create(test_user, { password = test_pass }) - - test:plan(5) - - local tube = queue.create_tube('test', 'fifo', { engine = engine }) - tube:put('help'); - local task = tube:take(); - test:is(task[1], 0, 'we can get record') - tube:release(task[1]) - - -- checking without grants - box.session.su('test') - local stat, er = pcall(tube.take, tube) - test:is(stat, false, 'we\'re getting error') - box.session.su('admin') - - -- checking with grants - tube:grant('test') - box.session.su('test') - local a = tube:take() - test:is(a[1], 0, 'we aren\'t getting any error') - local b = tube:take(0.1) - test:isnil(b, 'we aren\'t getting any error') - local c = tube:ack(a[1]) - test:is(a[1], 0, 'we aren\'t getting any error') - box.session.su('admin') - - -- checking double grants - tube:grant('test') - - box.schema.user.drop(test_user) - tube:drop() -end) - -test:test('check for call grants', function(test) - -- prepare for tests - rawset(_G, 'queue', require('queue')) - box.schema.user.create(test_user, { password = test_pass }) - - test:plan(12) - - local tube = queue.create_tube('test', 'fifo', { engine = engine }) - tube:put('help'); - local task = tube:take(); - test:is(task[1], 0, 'we can get record') - tube:release(task[1]) - - -- checking without grants - box.session.su('test') - local stat, er = pcall(tube.take, tube) - test:is(stat, false, 'we\'re getting error') - box.session.su('admin') - - -- checking with grants - tube:grant('test') - - box.session.su('test') - local a = tube:take() - test:is(a[1], 0, 'we aren\'t getting any error') - local b = tube:take(0.1) - test:isnil(b, 'we aren\'t getting any error') - local c = tube:release(a[1]) - test:is(a[1], 0, 'we aren\'t getting any error') - box.session.su('admin') - - local nb_connect = netbox.connect - if nb_connect == nil then - nb_connect = netbox.new - end - local con = nb_connect( - ('%s:%s@%s:%s'):format(test_user, test_pass, test_host, test_port) - ) - --[[ - local con = netbox.connect(uri.format({ - login = test_user, password = test_pass, - host = test_host, service = test_port, - }, true)) - ]]-- - - local stat, err = pcall(con.call, con, 'queue.tube.test:take') - test:is(stat, false, 'we\'re getting error') - - -- granting call - tube:grant('test', { call = true }) - - local qc_arg_unpack = function(arg) - if qc.check_version({1, 7}) then - return arg - end - return arg and arg[1] - end - - local a = con:call('queue.tube.test:take') - test:is(qc_arg_unpack(a[1]), 0, 'we aren\'t getting any error') - local b = con:call('queue.tube.test:take', qc.pack_args(0.1)) - test:isnil( - qc_arg_unpack(qc_arg_unpack(b)), - 'we aren\'t getting any error' - ) - local c = con:call('queue.tube.test:ack', qc.pack_args(qc_arg_unpack(a[1]))) - test:is(qc_arg_unpack(a[1]), 0, 'we aren\'t getting any error') - - local d = con:call('queue.tube.test:put', {'help'}) - test:is(qc_arg_unpack(d[1]), 0, 'we aren\'t getting any error') - - local e = con:call('queue.statistics') - test:is(type(qc_arg_unpack(e)), 'table', 'we aren\'t getting any error') - - tube:grant('test', { truncate = true }) - local f = con:call('queue.tube.test:truncate') - test:isnil(qc_arg_unpack(f), 'we aren\'t getting any error') - - -- check grants again - tube:grant('test', { call = true }) - - rawset(_G, 'queue', nil) - tube:drop() -end) - -test:test('check tube existence', function(test) - test:plan(14) - local queue = require('queue') - test:is(#queue.tube(), 0, 'checking for empty tube list') - assert(#queue.tube() == 0) - - local tube1 = queue.create_tube('test1', 'fifo', { engine = engine }) - test:is(#queue.tube(), 1, 'checking for not empty tube list') - - local tube2 = queue.create_tube('test2', 'fifo', { engine = engine }) - test:is(#queue.tube(), 2, 'checking for not empty tube list') - - test:is(queue.tube('test1'), true, 'checking for tube existence') - test:is(queue.tube('test2'), true, 'checking for tube existence') - test:is(queue.tube('test3'), false, 'checking for tube nonexistence') - - tube2:drop() - test:is(#queue.tube(), 1, 'checking for not empty tube list') - - test:is(queue.tube('test1'), true, 'checking for tube existence') - test:is(queue.tube('test2'), false, 'checking for tube nonexistence') - test:is(queue.tube('test3'), false, 'checking for tube nonexistence') - - tube1:drop() - test:is(#queue.tube(), 0, 'checking for empty tube list') - - test:is(queue.tube('test1'), false, 'checking for tube nonexistence') - test:is(queue.tube('test2'), false, 'checking for tube nonexistence') - test:is(queue.tube('test3'), false, 'checking for tube nonexistence') -end) - -tnt.finish() - -os.exit(test:check() and 0 or 1) --- vim: set ft=lua : diff --git a/t/100-limfifottl.t b/t/100-limfifottl.t deleted file mode 100755 index e78b0649..00000000 --- a/t/100-limfifottl.t +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env tarantool -local yaml = require('yaml') -local fiber = require('fiber') - -local queue = require('queue') -local state = require('queue.abstract.state') - -local engine = os.getenv('ENGINE') or 'memtx' - -local tnt = require('t.tnt') -tnt.cfg{} - -local test = require('tap').test() - -if engine == 'memtx' then - test:plan(9) - - test:ok(rawget(box, 'space'), 'box started') - test:ok(queue, 'queue is loaded') - - local tube = queue.create_tube('lim3_tube', 'limfifottl', { engine = engine, capacity = 3 }) - local unlim_tube = queue.create_tube('unlim_tube', 'limfifottl', { engine = engine }) - - test:ok(tube, 'test tube created') - test:is(tube.name, 'lim3_tube', 'tube.name') - test:is(tube.type, 'limfifottl', 'tube.type') - - test:test('Put timeout is reached', function(test) - test:plan(4) - - test:ok(tube:put{1}, 'task 1 was put') - test:ok(tube:put{2}, 'task 2 was put') - test:ok(tube:put{3}, 'task 3 was put') - test:is(tube:put({4}, {timeout = 0.1}), nil, 'task 4 wasn\'t put cause timeout') - end) - - test:test('Put after freeing up space', function(test) - test:plan(3) - local put_fiber = fiber.create(function() - test:ok(tube:put({4}, {timeout = 1}), 'task 4 was put') - end) - - local task = tube:take() - test:ok(task, 'task 3 was taken') - test:is(tube:ack(task[1])[2], state.DONE, 'task 3 is done') - - while put_fiber:status() ~= 'dead' do - fiber.sleep(.01) - end - end) - - test:test('Get current queue length', function(test) - test:plan(1) - test:is(tube.raw:len(), 3, 'tube length is 3') - end) - - test:test('Unlimited tube put', function(test) - test:plan(3) - - test:is(unlim_tube:take(0), nil, 'tube is empty') - test:ok(unlim_tube:put{1}, 'task 1 was put') - test:ok(unlim_tube:put{2}, 'task 2 was put') - end) -else - test:plan(1) - local ok = pcall(queue.create_tube, 'unsupported_engine', 'limfifottl', { engine = engine }) - test:ok(not ok, 'vinyl engine is not allowed') -end - -tnt.finish() -os.exit(test:check() and 0 or 1) --- vim: set ft=lua : diff --git a/t/110-disconnect-trigger-check.t b/t/110-disconnect-trigger-check.t deleted file mode 100755 index f5bc2ddf..00000000 --- a/t/110-disconnect-trigger-check.t +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env tarantool - -local fiber = require('fiber') -local netbox = require('net.box') -local os = require('os') -local queue = require('queue') -local tap = require('tap') -local tnt = require('t.tnt') - -local tube -local test = tap.test('lost a session id after yield') - --- The test cases are in check_result(). -test:plan(2) - --- Verify that _queue_taken space is empty. -local function check_result() - if tube == nil then - return - end - - -- tube:drop() is most simple way to check that _queue_taken - -- is empty. It give an error if it is not so. - local ok, res = pcall(tube.drop, tube) - test:is(ok, true, 'drop empty queue') - test:is(res, true, 'tube:drop() result is true') - - tnt.finish() -end - --- Yield in queue's on_disconnect trigger (which handles a client --- disconnection) may lead to a situation when _queue_taken --- temporary space is not cleaned and becomes inconsistent with --- 'status' field in space. This appears only on --- tarantool versions affected by gh-4627. --- --- See https://github.com/tarantool/queue/issues/103 --- See https://github.com/tarantool/tarantool/issues/4627 -local function test_lost_session_id_after_yield() - -- We must check the results of a test after - -- the queue._on_consumer_disconnect trigger - -- has been done. - -- - -- Triggers are run in LIFO order. - box.session.on_disconnect(check_result) - - local listen = 'localhost:1918' - tnt.cfg{listen = listen} - - local driver = 'fifottl' - tube = queue.create_tube('test_tube', driver, {if_not_exists = true}) - - rawset(_G, 'queue', require('queue')) - tube:grant('guest', {call = true}) - - -- We need at least two tasks to trigger box.session.id() - -- call after a yield in the queue._on_consumer_disconnect - -- trigger (in the version of queue before the fix). See - -- more below. - queue.tube.test_tube:put('1') - queue.tube.test_tube:put('2') - local connection = netbox.connect(listen) - connection:call('queue.tube.test_tube:take') - connection:call('queue.tube.test_tube:take') - - -- After disconnection of a client the _on_consumer_disconnect - -- trigger is run. It changes 'status' field for tuples in - -- space in a loop and removes the corresponding - -- tuples from _queue_taken space. The version before the fix - -- operates in this way: - -- - -- | <_on_consumer_disconnect> - -- | for task in tasks of the client: - -- | call - -- | - -- | - -- | delete _queue_taken tuple using box.session.id() - -- | update space using task_id -- !! yield - -- - -- So the deletion from _queue_taken may be unable to delete - -- right tuples for second and following tasks, because - -- box.session.id() may give a garbage. - connection:close() - - -- Wait for check_result() trigger, which will ensure that - -- _queue_taken space is cleaned and will exit successfully - -- in the case (or exit abnormally otherwise). - fiber.sleep(5) - - -- Wrong session id may lead to 'Task was not taken in the - -- session' error in the _on_consumer_disconnect and so the - -- second on_disconnect trigger (check_result) will not be - -- fired. - os.exit(test:check() and 0 or 1) -end - -test_lost_session_id_after_yield() --- vim: set ft=lua : diff --git a/t/120-take-task-after-reconnect.t b/t/120-take-task-after-reconnect.t deleted file mode 100755 index 44c49202..00000000 --- a/t/120-take-task-after-reconnect.t +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env tarantool - -local fiber = require('fiber') -local netbox = require('net.box') -local os = require('os') -local queue = require('queue') -local tap = require('tap') -local tnt = require('t.tnt') - - -local test = tap.test('take a task after reconnect') -test:plan(1) - -local listen = 'localhost:1918' -tnt.cfg{listen = listen} - - -local function test_take_task_after_disconnect(test) - test:plan(1) - local driver = 'fifottl' - local tube = queue.create_tube('test_tube', driver, - {if_not_exists = true}) - rawset(_G, 'queue', require('queue')) - tube:grant('guest', {call = true}) - local task_id = tube:put('test_data')[1] - -- Now we have one task in a ready state - - local connection = netbox.connect(listen) - local fiber_1 = fiber.create(function() - connection:call('queue.tube.test_tube:take') - connection:call('queue.tube.test_tube:take') - end) - - -- This is not a best practice but we need to use the fiber.sleep() - -- (not fiber.yield()). - -- Expected results from a sleep() calling: - -- 1) Execute first connection:call('queue.tube.test_tube:take') - -- Now one task in a taken state - -- 2) Call the second connection:call('queue.tube.test_tube:take') - -- and to hang the fiber_1 - -- 3) Start a fiber on the server side of connection which will execute - -- second queue.tube.test_tube:take call and hang because the queue - -- is empty - fiber.sleep(0.1) - - connection:close() - - fiber.sleep(0.1) - -- The taken task will be released (cause - disconnection). - -- After that the fiber which waiting of a ready task (into take procedure) - -- will try to take this task (before the fix). - - - test:is(tube:peek(task_id)[2] == 'r', true, 'Task in ready state') -end - - -test:test('Don\'t take a task after disconnect', test_take_task_after_disconnect) - - -tnt.finish() -os.exit(test:check() and 0 or 1) --- vim: set ft=lua : diff --git a/t/130-release-all-tasks-on-start.t b/t/130-release-all-tasks-on-start.t deleted file mode 100755 index d302963f..00000000 --- a/t/130-release-all-tasks-on-start.t +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env tarantool - -local os = require('os') -local queue = require('queue') -local tap = require('tap') -local tnt = require('t.tnt') - -local test = tap.test('release all tasks on start') -test:plan(1) - ---- --- Accept gh-66, we need to release all taken tasks on start. --- Instead of tarantool reboot, we will additionally call queue.start() --- to simulate the reload of the module. This is not a clean enough, --- because fibers launched by the module are not cleaned up. --- See https://github.com/tarantool/queue/issues/66 --- --- All tricks in this test are done by professionals, don't try --- to repeat it yourself!!! -local function check_release_tasks_on_start() - tnt.cfg() - -- The "fifottl" driver was choosen for check gh-121. - -- We cann't use opts == nil as argument for driver "release" - -- method. This is the policy of the module "queue" (check - -- opts in abstract.lua, instead to check inside the driver). - -- See https://github.com/tarantool/queue/issues/121 - local driver = 'fifottl' - local tube = queue.create_tube('test_tube', driver) - - tube:put('1') - tube:put('2') - tube:put('3') - - tube:take() - tube:take() - tube:take() - - -- Simulate the module reload. - queue.start() - - local ready_tasks_num = queue.statistics()['test_tube']['tasks']['ready'] - test:is(ready_tasks_num, 3, 'check release tasks on start') -end - -check_release_tasks_on_start() - -tnt.finish() -os.exit(test:check() and 0 or 1) --- vim: set ft=lua : diff --git a/t/140-register-driver-after-cfg.t b/t/140-register-driver-after-cfg.t deleted file mode 100755 index 7c734e52..00000000 --- a/t/140-register-driver-after-cfg.t +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env tarantool -local tap = require('tap') -local tnt = require('t.tnt') - -local test = tap.test('test driver register') -test:plan(3) - -local mock_tube = { - create_space = function() end, - new = function() end -} - --- As opposed to 001-tube-init.t, queue initialization --- and driver registration are done after cfg(). -local function check_driver_register() - tnt.cfg() - local queue = require('queue') - queue.register_driver('mock', mock_tube) - test:is(queue.driver.mock, mock_tube, 'driver has been registered') - - local standart_drivers = { - 'fifo', - 'fifottl', - 'limfifottl', - 'utube', - 'utubettl' - } - local check_standart_drivers = true - - for _, v in pairs(standart_drivers) do - if queue.driver[v] == nil then - check_standart_drivers = false - break - end - end - - test:ok(check_standart_drivers, 'standard drivers are defined') - - local res, err = pcall(queue.register_driver, 'mock', mock_tube) - local check = not res and - string.match(err, 'overriding registered driver') ~= nil - test:ok(check, 'check a driver override failure') -end - -check_driver_register() - -tnt.finish() -os.exit(test:check() and 0 or 1) --- vim: set ft=lua : diff --git a/t/150-lazy-start.t b/t/150-lazy-start.t deleted file mode 100755 index 1001d4e7..00000000 --- a/t/150-lazy-start.t +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env tarantool -local tap = require('tap') -local tnt = require('t.tnt') - -local test = tap.test('test driver register') -test:plan(3) - -local function check_lazy_start() - -- Needed for bootstrap - tnt.cfg{} - - tnt.cfg{read_only = true} - local queue = require('queue') - - local err_msg = 'Please configure box.cfg{} in read/write mode first' - local res, err = pcall(function() queue.stats() end) - local check = not res and string.match(err,err_msg) ~= nil - test:ok(check, 'check queue delayed start') - - tnt.cfg({read_only = true}) - res, err = pcall(function() queue.stats() end) - check = not res and string.match(err, err_msg) ~= nil - test:ok(check, 'check box reconfiguration with read_only = true') - - tnt.cfg({read_only = false}) - res = pcall(function() queue.stats() end) - test:ok(res, 'queue has been started') -end - -check_lazy_start() - -tnt.finish() -os.exit(test:check() and 0 or 1) --- vim: set ft=lua : diff --git a/t/160-validate-space.t b/t/160-validate-space.t deleted file mode 100755 index cca151c4..00000000 --- a/t/160-validate-space.t +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env tarantool - -local tap = require('tap') -local os = require('os') -local tnt = require('t.tnt') -local test = tap.test('test space validation') - -local fifo = require('queue.abstract.driver.fifo') -local fifottl = require('queue.abstract.driver.fifottl') -local utube = require('queue.abstract.driver.utube') -local utubettl = require('queue.abstract.driver.utubettl') - -local engine = os.getenv('ENGINE') or 'memtx' - -test:plan(8) -tnt.cfg{} - -local function test_corrupted_space(test, driver, indexes) - test:plan(table.getn(indexes)) - - -- Can't drop primary key in space while secondary keys exist. - -- So, drop other indexes previously. - local function remove_task_id_index(space, indexes) - for _, index in pairs(indexes) do - if index ~= 'task_id' then - space.index[index]:drop() - end - end - space.index.task_id:drop() - end - - for _, index in pairs(indexes) do - test:test(index .. ' index does not exist', function(test) - test:plan(2) - - local space = driver.create_space('corrupted_space', - {engine = engine}) - - if index == 'task_id' then - remove_task_id_index(space, indexes) - else - space.index[index]:drop() - end - - local res, err = pcall(driver.new, space) - local err_match_msg = string.format('space "corrupted_space"' .. - ' does not have "%s" index', index) - test:ok(not res, 'exception was thrown') - test:ok(err:match(err_match_msg) ~= nil, 'text of exception') - - space:drop() - end) - end -end - -local function test_name_conflict(test, driver) - test:plan(2) - - local conflict_space = box.schema.create_space('conflict_tube') - local res, err = pcall(driver.create_space,'conflict_tube', - {engine = engine, if_not_exists = true}) - - test:ok(not res, 'exception was thrown') - test:ok(err:match('space "conflict_tube" does not' .. - ' have "task_id" index') ~= nil, 'text of exception') - - conflict_space:drop() -end - -test:test('test corrupted space fifo', function(test) - test_corrupted_space(test, fifo, {'task_id', 'status'}) -end) - -test:test('test corrupted space fifottl', function(test) - test_corrupted_space(test, fifottl, {'task_id', 'status', 'watch'}) -end) - -test:test('test corrupted space utube', function(test) - test_corrupted_space(test, utube, {'task_id', 'status', 'utube'}) -end) - -test:test('test corrupted space utubettl', function(test) - test_corrupted_space(test, utubettl, - {'task_id', 'status', 'utube', 'watch', 'utube_pri'}) -end) - -test:test('Space name conflict fifo', function(test) - test_name_conflict(test, fifo) -end) - -test:test('Space name conflict fifo', function(test) - local fifo = require('queue.abstract.driver.fifo') - test_name_conflict(test, fifottl) -end) - -test:test('Space name conflict fifo', function(test) - local fifo = require('queue.abstract.driver.fifo') - test_name_conflict(test, utube) -end) - -test:test('Space name conflict fifo', function(test) - local fifo = require('queue.abstract.driver.fifo') - test_name_conflict(test, utubettl) -end) - -tnt.finish() -os.exit(test:check() and 0 or 1) --- vim: set ft=lua : diff --git a/t/170-register-driver-after-reload.t b/t/170-register-driver-after-reload.t deleted file mode 100755 index 20ff97f8..00000000 --- a/t/170-register-driver-after-reload.t +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env tarantool - -local os = require('os') -local queue = require('queue') -local tap = require('tap') -local tnt = require('t.tnt') - -local test = tap.test('custom driver registration after reload') -test:plan(1) - -tnt.cfg() - ---- Accept gh-137, we need to check custom driver registration --- after restart. Instead of tarantool reboot, we will additionally --- call queue.start() to simulate the reload of the module. This --- is not a clean enough, because queue module doesn't provide the --- hot restart. --- --- All tricks in this test are done by professionals, don't try --- to repeat it yourself!!! -local function check_driver_registration_after_reload() - local fifo = require('queue.abstract.driver.fifo') - queue.register_driver('fifo_cust', fifo) - - local tube = queue.create_tube('tube_cust', 'fifo_cust') - tube:put('1') - local task_id = tube:take()[1] - - -- Simulate the module reload. - queue.driver.fifo_cust = nil - queue.start() - - -- Check the task has been released after reload. - queue.register_driver('fifo_cust', fifo) - local task_status = queue.tube.tube_cust:peek(task_id)[2] - test:is(task_status, 'r', 'check driver registration after reload') -end - -check_driver_registration_after_reload() - -tnt.finish() -os.exit(test:check() and 0 or 1) --- vim: set ft=lua : diff --git a/t/180-work-with-uuid.t b/t/180-work-with-uuid.t deleted file mode 100755 index beb25633..00000000 --- a/t/180-work-with-uuid.t +++ /dev/null @@ -1,189 +0,0 @@ -#!/usr/bin/env tarantool - -local fiber = require('fiber') -local netbox = require('net.box') -local tap = require('tap') -local os = require('os') -local tnt = require('t.tnt') - -local test = tap.test('test work with uuid') -test:plan(4) - -local listen = 'localhost:1918' -tnt.cfg{ listen = listen } - -rawset(_G, 'queue', require('queue')) - -test:test('test UUID format validation', function(test) - test:plan(1) - - -- We need to call the `grant()` to have sufficient rights - -- to call `identify.` - local tube = queue.create_tube('test_tube', 'fifo') - tube:grant('guest', { call = true }) - - local conn = netbox.connect(listen) - local invalid_uuid = 'just-for-fun' - local ok, err = pcall(conn.call, conn, 'queue.identify', {invalid_uuid}) - - local check_validation = not ok and err:match('Invalid UUID format.') - test:ok(check_validation, 'UUID format validation has been checked.') - conn:close() - - tube:drop() -end) - -test:test('test work with two consumers with the same uuid', function(test) - test:plan(4) - - -- Preparing of the tube. - local tube = queue.create_tube('test_tube', 'fifo') - tube:grant('guest', { call = true }) - local uuid_con1 - local uuid_con2 - tube:put('test_data') - - local cond = fiber.cond() - - local fiber_consumer_1 = fiber.new(function() - local conn = netbox.connect(listen) - local task = conn:call('queue.tube.test_tube:take') - local task_id = task[1] - uuid_con1 = conn:call('queue.identify') - - -- Wait until consumer 2 connects to the session. - cond:signal() - cond:wait() - - -- Reconnect to the session. - conn:close() - conn = netbox.connect(listen) - uuid_con1 = conn:call('queue.identify', {uuid_con2}) - test:ok(uuid_con1 == uuid_con2, - 'reconnection of consumer 1 has been completed') - - -- Ack the task and close the connection. - task = conn:call('queue.tube.test_tube:ack', {task_id}) - test:ok(task[1] == task_id and task[2] == '-', 'task has been acked') - conn:close() - - -- Wakeup the consumer 2 fiber and wait for it to finishes. - cond:signal() - cond:wait() - end) - - fiber_consumer_1:set_joinable(true) - - local fiber_consumer_2 = fiber.create(function() - -- Waiting for consumer 1 identification - cond:wait() - - -- Connect to server and connect to the same session as consumer 1. - local conn = netbox.connect(listen) - uuid_con2 = conn:call('queue.identify', {uuid_con1}) - test:ok(uuid_con1 == uuid_con2, - 'consumer 2 identification completed correctly') - - -- Wait until consumer 1 will reconnect and "ack" the task. - cond:signal() - cond:wait() - - -- Close the connection and wakeup the consumer 1 fiber. - conn:close() - cond:signal() - end) - - -- Wait for consumers fibers to finishes. - local ok = fiber_consumer_1:join() - test:ok(ok, 'reconnection test done') - - tube:drop() -end) - -test:test('test reconnect and ack the task', function(test) - test:plan(2) - - -- Preparing of the tube. - queue.cfg({ ttr = 60 }) - local tube = queue.create_tube('test_tube', 'fifo') - tube:put('test_data') - tube:grant('guest', { call = true }) - - local fiber_consumer = fiber.new(function() - -- Connect and take a task. - local conn = netbox.connect(listen) - local task = conn:call('queue.tube.test_tube:take') - local task_id = task[1] - local session_id = conn:call('queue.identify') - conn:close() - - -- Reconnect and ack the task. - conn = netbox.connect(listen) - conn:call('queue.identify', {session_id}) - task = conn:call('queue.tube.test_tube:ack', {task_id}) - test:ok(task[1] == task_id and task[2] == '-', 'task has been acked') - - conn:close() - end) - - fiber_consumer:set_joinable(true) - - local ok = fiber_consumer:join() - test:ok(ok, 'reconnect and ack the task test done') - - tube:drop() -end) - -test:test('test expiration', function(test) - test:plan(5) - - -- Preparing of the tube. - local tube = queue.create_tube('test_tube', 'fifo') - tube:put('test_data') - tube:grant('guest', { call = true }) - - local fiber_consumer = fiber.new(function() - queue.cfg({ ttr = 1 }) - -- Connect and take a task. - local conn = netbox.connect(listen) - local task_id = conn:call('queue.tube.test_tube:take')[1] - local uuid_con = conn:call('queue.identify') - conn:close() - - -- Check that the task is in a "taken" state before ttr expires. - fiber.sleep(0.1) - test:ok(tube:peek(task_id)[2] == 't', 'task in taken state') - fiber.sleep(2) - - -- The task must be released after the ttr expires. - test:ok(tube:peek(task_id)[2] == 'r', 'task in ready state after ttr') - - -- The old queue session must expire. - conn = netbox.connect(listen) - local ok, err = pcall(conn.call, conn, 'queue.identify', - {uuid_con}) - local check_ident = not ok and err:match('UUID .* is unknown.') - test:ok(check_ident, 'the old queue session has expired.') - conn:close() - - -- If ttr = 0, the task should be released immediately. - queue.cfg({ ttr = 0 }) - conn = netbox.connect(listen) - task_id = conn:call('queue.tube.test_tube:take')[1] - conn:close() - fiber.sleep(0.1) - test:ok(tube:peek(task_id)[2] == 'r', - 'task has been immediately released') - end) - - fiber_consumer:set_joinable(true) - - local ok = fiber_consumer:join() - test:ok(ok, 'expiration test done') - - tube:drop() -end) - -tnt.finish() -os.exit(test:check() and 0 or 1) --- vim: set ft=lua : diff --git a/t/190-work-with-ttl-buried-task.t b/t/190-work-with-ttl-buried-task.t deleted file mode 100755 index 8eb5652f..00000000 --- a/t/190-work-with-ttl-buried-task.t +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env tarantool - -local os = require('os') - -local fiber = require('fiber') - -local queue = require('queue') -local state = require('queue.abstract.state') - -local tap = require('tap') -local tnt = require('t.tnt') - -local test = tap.test('work with "ttl" of buried task.') -test:plan(1) - --- Fields in the task tuple. -local TASK_ID = 1 -local TASK_STATE = 2 - -tnt.cfg{} - -test:test('test work with "ttl", when "bury" after "take"', function(test) - -- Before the patch if a task has been "buried" after it was "taken" - -- (and the task has "ttr") when the time in `i_next_event` will be - -- interpreted as "ttl" in `{fifottl,utubettl}_fiber_iteration` and - -- the task will be deleted. - local drivers = {'fifottl', 'utubettl'} - test:plan(3 * table.getn(drivers)) - - local TTR = 0.2 - local TTL = 1 - - for _, driver in pairs(drivers) do - local tube = queue.create_tube('test_tube', driver, {if_not_exists = true}) - local task = tube:put('task1', {ttl = TTL, ttr = TTR}) - - -- "Take" a task and "bury" it. - task = tube:take(0) - local id = task[TASK_ID] - tube:bury(id) - - -- Check status of the task. - task = tube:peek(id) - test:is(task[TASK_STATE], state.BURIED, - ('task "buried", driver: "%s"'):format(driver)) - - -- Check status of the task after "ttr" has expired. - fiber.sleep(TTR * 2) - task = tube:peek(id) - test:is(task[TASK_STATE], state.BURIED, - ('task is still "buried", driver: "%s"'):format(driver)) - - -- Check status of the task after "ttl" has expired. - fiber.sleep(TTL * 2) - local ok, res = pcall(tube.peek, tube, id) - test:ok(res:match(string.format('Task %d not found', id)), - ('task done, driver: "%s"'):format(driver)) - - tube:drop() - end -end) - -tnt.finish() -os.exit(test:check() and 0 or 1) - --- vim: set ft=lua : diff --git a/t/200-master-replica.t b/t/200-master-replica.t deleted file mode 100755 index 6cc709f2..00000000 --- a/t/200-master-replica.t +++ /dev/null @@ -1,398 +0,0 @@ -#!/usr/bin/env tarantool - -local fio = require('fio') -local log = require('log') -local tnt = require('t.tnt') -local test = require('tap').test('') -local uuid = require('uuid') -local queue = require('queue') -local fiber = require('fiber') - -local session = require('queue.abstract.queue_session') -local queue_state = require('queue.abstract.queue_state') -rawset(_G, 'queue', require('queue')) - -local qc = require('queue.compat') -if not qc.check_version({2, 4, 1}) then - log.info('Tests skipped, tarantool version < 2.4.1') - return -end - --- Replica connection handler. -local conn = {} - -test:plan(8) - -test:test('Check master-replica setup', function(test) - test:plan(9) - local engine = os.getenv('ENGINE') or 'memtx' - tnt.cluster.cfg{} - - test:ok(rawget(box, 'space'), 'box started') - test:ok(queue, 'queue is loaded') - - test:ok(tnt.cluster.wait_replica(), 'wait for replica to connect') - conn = tnt.cluster.connect_replica() - test:ok(conn.error == nil, 'no errors on connect to replica') - test:ok(conn:ping(), 'ping replica') - test:is(queue.state(), 'RUNNING', 'check master queue state') - conn:eval('rawset(_G, "queue", require("queue"))') - test:is(conn:call('queue.state'), 'INIT', 'check replica queue state') - - -- Setup tube. Set ttr = 0.5 for sessions expire testing. - conn:call('queue.cfg', {{ttr = 0.5, in_replicaset = true}}) - test:isnil(conn:call('queue.create_tube', {'test', 'fifo'}), - 'check api call in INIT state') - queue.cfg{ttr = 0.5, in_replicaset = true} - local tube = queue.create_tube('test', 'fifo', {engine = engine}) - test:ok(tube, 'test tube created') -end) - -test:test('Check queue state switching', function(test) - test:plan(4) - box.cfg{read_only = true} - test:ok(queue_state.poll(queue_state.states.WAITING, 10), - "queue state changed to waiting") - test:is(session.expiration_fiber:status(), 'dead', - "check that session expiration fiber is canceled") - box.cfg{read_only = false} - test:ok(queue_state.poll(queue_state.states.RUNNING, 10), - "queue state changed to running") - test:is(session.expiration_fiber:status(), 'suspended', - "check that session expiration fiber started") -end) - -test:test('Check session resuming', function(test) - test:plan(16) - local client = tnt.cluster.connect_master() - test:ok(client.error == nil, 'no errors on client connect to master') - local session_uuid = client:call('queue.identify') - local uuid_obj = uuid.frombin(session_uuid) - - test:ok(queue.tube.test:put('testdata'), 'put task') - local task_master = client:call('queue.tube.test:take') - test:ok(task_master, 'task was taken') - test:is(task_master[3], 'testdata', 'task.data') - - local qt = box.space._queue_taken_2:select() - test:is(uuid.frombin(qt[1][4]):str(), uuid_obj:str(), - 'task taken by actual uuid') - - -- Switch roles. - box.cfg{read_only = true} - queue_state.poll(queue_state.states.WAITING, 10) - test:is(queue.state(), 'WAITING', 'master state is waiting') - conn:eval('box.cfg{read_only=false}') - conn:eval([[ - queue_state = require('queue.abstract.queue_state') - queue_state.poll(queue_state.states.RUNNING, 10) - ]]) - test:is(conn:call('queue.state'), 'RUNNING', 'replica state is running') - - local cfg = conn:eval('return queue.cfg') - test:is(cfg.ttr, 0.5, 'check cfg applied after lazy start') - - test:ok(conn:call('queue.identify', {session_uuid}), 'identify old session') - local stat = conn:call('queue.statistics') - test:is(stat.test.tasks.taken, 1, 'taken tasks count') - test:is(stat.test.tasks.done, 0, 'done tasks count') - local task_replica = conn:call('queue.tube.test:ack', {task_master[1]}) - test:is(task_replica[3], 'testdata', 'check task data') - local stat = conn:call('queue.statistics') - test:is(stat.test.tasks.taken, 0, 'taken tasks count after ack()') - test:is(stat.test.tasks.done, 1, 'done tasks count after ack()') - - -- Switch roles back. - conn:eval('box.cfg{read_only=true}') - conn:eval([[ - queue_state = require('queue.abstract.queue_state') - queue_state.poll(queue_state.states.WAITING, 10) - ]]) - box.cfg{read_only = false} - queue_state.poll(queue_state.states.RUNNING, 10) - test:is(queue.state(), 'RUNNING', 'master state is running') - test:is(conn:call('queue.state'), 'WAITING', 'replica state is waiting') - client:close() -end) - -test:test('Check session resuming (client disconnected)', function(test) - test:plan(17) - local client = tnt.cluster.connect_master() - test:ok(client.error == nil, 'no errors on client connect to master') - local session_uuid = client:call('queue.identify') - local uuid_obj = uuid.frombin(session_uuid) - - test:ok(queue.tube.test:put('testdata'), 'put task') - local task_master = client:call('queue.tube.test:take') - test:ok(task_master, 'task was taken') - test:is(task_master[3], 'testdata', 'task.data') - client:close() - - local qt = box.space._queue_taken_2:select() - test:is(uuid.frombin(qt[1][4]):str(), uuid_obj:str(), - 'task taken by actual uuid') - - -- Wait for disconnect callback. - local attempts = 0 - while true do - local tuple = box.space._queue_shared_sessions:get(session_uuid) - - if tuple then - test:is(uuid.frombin(tuple[1]):str(), uuid_obj:str(), - 'check inactive sessions') - break - end - - attempts = attempts + 1 - if attempts == 10 then - test:ok(false, 'check inactive sessions') - return false - end - fiber.sleep(0.01) - end - - -- Switch roles. - box.cfg{read_only = true} - queue_state.poll(queue_state.states.WAITING, 10) - test:is(queue.state(), 'WAITING', 'master state is waiting') - conn:eval('box.cfg{read_only=false}') - conn:eval([[ - queue_state = require('queue.abstract.queue_state') - queue_state.poll(queue_state.states.RUNNING, 10) - ]]) - test:is(conn:call('queue.state'), 'RUNNING', 'replica state is running') - - local cfg = conn:eval('return queue.cfg') - test:is(cfg.ttr, 0.5, 'check cfg applied after lazy start') - - test:ok(conn:call('queue.identify', {session_uuid}), 'identify old session') - local stat = conn:call('queue.statistics') - test:is(stat.test.tasks.taken, 1, 'taken tasks count') - test:is(stat.test.tasks.done, 1, 'done tasks count') - local task_replica = conn:call('queue.tube.test:ack', {task_master[1]}) - test:is(task_replica[3], 'testdata', 'check task data') - local stat = conn:call('queue.statistics') - test:is(stat.test.tasks.taken, 0, 'taken tasks count after ack()') - test:is(stat.test.tasks.done, 2, 'done tasks count after ack()') - - -- Switch roles back. - conn:eval('box.cfg{read_only=true}') - conn:eval([[ - queue_state = require('queue.abstract.queue_state') - queue_state.poll(queue_state.states.WAITING, 10) - ]]) - box.cfg{read_only = false} - queue_state.poll(queue_state.states.RUNNING, 10) - test:is(queue.state(), 'RUNNING', 'master state is running') - test:is(conn:call('queue.state'), 'WAITING', 'replica state is waiting') -end) - -test:test('Check task is cleaned after migrate', function(test) - test:plan(8) - local client = tnt.cluster.connect_master() - local session_uuid = client:call('queue.identify') - local uuid_obj = uuid.frombin(session_uuid) - test:ok(queue.tube.test:put('testdata'), 'put task') - test:ok(client:call('queue.tube.test:take'), 'take task from master') - - -- Switch roles. - box.cfg{read_only = true} - - queue_state.poll(queue_state.states.WAITING, 10) - test:is(queue.state(), 'WAITING', 'master state is waiting') - conn:eval('box.cfg{read_only=false}') - conn:eval([[ - queue_state = require('queue.abstract.queue_state') - queue_state.poll(queue_state.states.RUNNING, 10) - ]]) - test:is(conn:call('queue.state'), 'RUNNING', 'replica state is running') - - -- Check task. - local stat = conn:call('queue.statistics') - test:is(stat.test.tasks.taken, 1, 'taken tasks count before timeout') - fiber.sleep(1.5) - local stat = conn:call('queue.statistics') - test:is(stat.test.tasks.taken, 0, 'taken tasks count after timeout') - - -- Switch roles back. - conn:eval('box.cfg{read_only=true}') - conn:eval([[ - queue_state = require('queue.abstract.queue_state') - queue_state.poll(queue_state.states.WAITING, 10) - ]]) - box.cfg{read_only = false} - queue_state.poll(queue_state.states.RUNNING, 10) - test:is(queue.state(), 'RUNNING', 'master state is running') - test:is(conn:call('queue.state'), 'WAITING', 'replica state is waiting') - client:close() -end) - -test:test('Check task is cleaned after migrate (client disconnected)', function(test) - test:plan(9) - local client = tnt.cluster.connect_master() - local session_uuid = client:call('queue.identify') - local uuid_obj = uuid.frombin(session_uuid) - test:ok(queue.tube.test:put('testdata'), 'put task') - test:ok(client:call('queue.tube.test:take'), 'take task from master') - client:close() - - -- Wait for disconnect callback. - local attempts = 0 - while true do - local tuple = box.space._queue_shared_sessions:get(session_uuid) - - if tuple then - test:is(uuid.frombin(tuple[1]):str(), uuid_obj:str(), - 'check inactive sessions') - break - end - - attempts = attempts + 1 - if attempts == 10 then - test:ok(false, 'check inactive sessions') - return false - end - fiber.sleep(0.01) - end - - -- Switch roles. - box.cfg{read_only = true} - - queue_state.poll(queue_state.states.WAITING, 10) - test:is(queue.state(), 'WAITING', 'master state is waiting') - conn:eval('box.cfg{read_only=false}') - conn:eval([[ - queue_state = require('queue.abstract.queue_state') - queue_state.poll(queue_state.states.RUNNING, 10) - ]]) - test:is(conn:call('queue.state'), 'RUNNING', 'replica state is running') - - -- Check task. - local stat = conn:call('queue.statistics') - test:is(stat.test.tasks.taken, 1, 'taken tasks count before timeout') - fiber.sleep(1.5) - local stat = conn:call('queue.statistics') - test:is(stat.test.tasks.taken, 0, 'taken tasks count after timeout') - - -- Switch roles back. - conn:eval('box.cfg{read_only=true}') - conn:eval([[ - queue_state = require('queue.abstract.queue_state') - queue_state.poll(queue_state.states.WAITING, 10) - ]]) - box.cfg{read_only = false} - queue_state.poll(queue_state.states.RUNNING, 10) - test:is(queue.state(), 'RUNNING', 'master state is running') - test:is(conn:call('queue.state'), 'WAITING', 'replica state is waiting') -end) - -test:test('Check in_replicaset switching', function(test) - test:plan(17) - test:ok(queue.tube.test:put('testdata0'), 'put task #0') - test:ok(queue.tube.test:put('testdata1'), 'put task #1') - local client = tnt.cluster.connect_master() - test:ok(client:call('queue.tube.test:take'), 'take task') - test:ok(client:call('queue.tube.test:take'), 'take task') - client:close() - -- Wait for disconnect callback. - local attempts = 0 - while true do - if #box.space._queue_shared_sessions:select() == 1 then - break - end - - attempts = attempts + 1 - if attempts == 10 then - test:ok(false, 'check inactive sessions') - return false - end - fiber.sleep(0.01) - end - test:is(box.space._queue_shared_sessions.temporary, false, - '_queue_shared_sessions is not temporary') - test:is(box.space._queue_taken_2.temporary, false, - '_queue_taken_2 is not temporary') - test:is(#box.space._queue_taken_2:select(), 2, - 'check _queue_taken_2 data') - queue.cfg{in_replicaset = false} - test:is(box.space._queue_shared_sessions.temporary, true, - '_queue_shared_sessions is temporary') - test:is(box.space._queue_taken_2.temporary, true, - '_queue_taken_2 is temporary') - test:is(#box.space._queue_taken_2:select(), 2, - 'check _queue_taken_2 data') - test:is(#box.space._queue_shared_sessions:select(), 1, - 'check _queue_shared_sessions data') - queue.cfg{in_replicaset = true} - test:is(box.space._queue_shared_sessions.temporary, false, - '_queue_shared_sessions is not temporary') - test:is(box.space._queue_taken_2.temporary, false, - '_queue_taken_2 is not temporary') - test:is(#box.space._queue_taken_2:select(), 2, - 'check _queue_taken_2 data') - test:is(#box.space._queue_shared_sessions:select(), 1, - 'check _queue_shared_sessions data') - test:is(queue.statistics().test.tasks.taken, 2, - 'taken tasks count before release_all') - queue.tube.test:release_all() - test:is(queue.statistics().test.tasks.taken, 0, - 'taken tasks count after release_all') -end) - --- gh-202 -test:test('Check that tubes indexes is actual after role change', function(test) - local engine = os.getenv('ENGINE') or 'memtx' - test:plan(10) - box.cfg{read_only = true} - queue_state.poll(queue_state.states.WAITING, 10) - test:is(queue.state(), 'WAITING', 'master state is waiting') - conn:eval('box.cfg{read_only=false}') - conn:eval([[ - queue_state = require('queue.abstract.queue_state') - queue_state.poll(queue_state.states.RUNNING, 10) - ]]) - test:is(conn:call('queue.state'), 'RUNNING', 'replica state is running') - conn:eval([[queue.create_tube('repl_tube', 'fifo', {engine =]] .. engine .. [[})]]) - - -- Switch roles back. - conn:eval('box.cfg{read_only=true}') - conn:eval([[ - queue_state = require('queue.abstract.queue_state') - queue_state.poll(queue_state.states.WAITING, 10) - ]]) - box.cfg{read_only = false} - queue_state.poll(queue_state.states.RUNNING, 10) - test:is(queue.state(), 'RUNNING', 'master state is running') - test:is(conn:call('queue.state'), 'WAITING', 'replica state is waiting') - test:ok(queue.tube.repl_tube, 'repl_tube is accessible') - - box.cfg{read_only = true} - queue_state.poll(queue_state.states.WAITING, 10) - test:is(queue.state(), 'WAITING', 'master state is waiting') - conn:eval('box.cfg{read_only=false}') - conn:eval([[ - queue_state = require('queue.abstract.queue_state') - queue_state.poll(queue_state.states.RUNNING, 10) - ]]) - test:is(conn:call('queue.state'), 'RUNNING', 'replica state is running') - conn:eval('queue.tube.repl_tube:drop()') - - -- Switch roles back. - conn:eval('box.cfg{read_only=true}') - conn:eval([[ - queue_state = require('queue.abstract.queue_state') - queue_state.poll(queue_state.states.WAITING, 10) - ]]) - box.cfg{read_only = false} - queue_state.poll(queue_state.states.RUNNING, 10) - test:is(queue.state(), 'RUNNING', 'master state is running') - test:is(conn:call('queue.state'), 'WAITING', 'replica state is waiting') - test:isnil(queue.tube.repl_tube, "repl_tube is not indexed") -end) - -rawset(_G, 'queue', nil) -conn:eval('rawset(_G, "queue", nil)') -conn:close() -tnt.finish() -os.exit(test:check() and 0 or 1) --- vim: set ft=lua : diff --git a/t/210-cfg-in-replicaset.t b/t/210-cfg-in-replicaset.t deleted file mode 100755 index 5a57a4f9..00000000 --- a/t/210-cfg-in-replicaset.t +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env tarantool - -local test = require('tap').test('') -local queue = require('queue') - -rawset(_G, 'queue', require('queue')) - -test:plan(2) - -test:test('Check in_replicaset switches', function(test) - test:plan(4) - local engine = os.getenv('ENGINE') or 'memtx' - - box.cfg({}) - - local status, err = pcall(queue.cfg, {in_replicaset = true}) - test:ok(status, 'in_replicaset = true switched') - local status, _ = pcall(queue.cfg, {in_replicaset = false}) - test:ok(status, 'in_replicaset = false switched') - local status, _ = pcall(queue.cfg, {in_replicaset = true}) - test:ok(status, 'in_replicaset = true switched') - local status, _ = pcall(queue.cfg, {in_replicaset = false}) - test:ok(status, 'in_replicaset = false switched') -end) - -test:test('Check error create temporary tube', function(test) - test:plan(2) - local engine = os.getenv('ENGINE') or 'memtx' - - box.cfg({}) - - queue.cfg{ttr = 0.5, in_replicaset = true} - local opts = {temporary = true, engine = engine} - local status, err = pcall(queue.create_tube, 'test', 'fifo', opts) - test:is(status, false, 'test tube should not be created') - local founded = string.find(err, - 'Cannot create temporary tube in replicaset mode') - test:ok(founded, 'unexpected error') -end) - -rawset(_G, 'queue', nil) -os.exit(test:check() and 0 or 1) --- vim: set ft=lua : diff --git a/t/220-mvcc.t b/t/220-mvcc.t deleted file mode 100644 index 8be053f3..00000000 --- a/t/220-mvcc.t +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env tarantool - -local qc = require('queue.compat') -local log = require('log') -if not qc.check_version({2, 6, 1}) then - log.info('Tests skipped, tarantool version < 2.6.1') - return -end - -local fiber = require('fiber') - -local test = require('tap').test() -test:plan(6) - -local queue = require('queue') - -local tnt = require('t.tnt') -tnt.cfg{memtx_use_mvcc_engine = true} - -local engine = 'memtx' - -test:ok(rawget(box, 'space'), 'box started') -test:ok(queue, 'queue is loaded') - -local tube = queue.create_tube('test', 'fifo', { engine = engine, temporary = false}) -test:ok(tube, 'test tube created') -test:is(tube.name, 'test', 'tube.name') -test:is(tube.type, 'fifo', 'tube.type') - ---- That test checks that https://github.com/tarantool/queue/pull/211 is fixed. --- Previously trying to make parallel 'put' or 'take' calls failed with mvcc enabled. -test:test('concurent put and take with mvcc', function(test) - test:plan(6) - -- channels are used to wait for fibers to finish - -- and check results of the 'take'/'put'. - local channel_put = fiber.channel(2) - test:ok(channel_put, 'channel created') - local channel_take = fiber.channel(2) - test:ok(channel_take, 'channel created') - - for i = 1, 2 do - fiber.create(function(i) - local err = pcall(tube.put, tube, i) - channel_put:put(err) - end, i) - end - - for i = 1, 2 do - local res = channel_put:get(1) - test:ok(res, 'task ' .. i .. ' was put') - end - - for i = 1, 2 do - fiber.create(function(i) - local err = pcall(tube.take, tube) - channel_take:put(err) - end, i) - end - - for i = 1, 2 do - local res = channel_take:get() - test:ok(res, 'task ' .. i .. ' was taken') - end -end) - -tnt.finish() -os.exit(test:check() and 0 or 1) - --- vim: set ft=lua: diff --git a/t/230-orphan-not-stalling-init.t b/t/230-orphan-not-stalling-init.t deleted file mode 100755 index d346ce19..00000000 --- a/t/230-orphan-not-stalling-init.t +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env tarantool - -local test = require('tap').test('') -local queue = require('queue') -local tnt = require('t.tnt') -local fio = require('fio') -local fiber = require('fiber') - -rawset(_G, 'queue', require('queue')) - -local qc = require('queue.compat') -if not qc.check_version({2, 10, 0}) then - require('log').info('Tests skipped, tarantool version < 2.10.0 ' .. - 'does not support the lazy init') - return -end - -local snapdir_optname = qc.snapdir_optname -local logger_optname = qc.logger_optname - -test:plan(1) - -test:test('Check orphan mode not stalling queue', function(test) - test:plan(4) - local engine = os.getenv('ENGINE') or 'memtx' - tnt.cluster.cfg{} - - local dir_replica = fio.tempdir() - local cmd_replica = { - arg[-1], - '-e', - [[ - box.cfg { - replication = { - 'replicator:password@127.0.0.1:3399', - 'replicator:password@127.0.0.1:3398', - }, - listen = '127.0.0.1:3396', - wal_dir = ']] .. dir_replica .. '\'' .. - ',' .. snapdir_optname() .. ' = \'' .. dir_replica .. '\'' .. - ',' .. logger_optname() .. ' = \'' .. - fio.pathjoin(dir_replica, 'tarantool.log') .. '\'' .. - '}' - } - - replica = require('popen').new(cmd_replica, { - stdin = 'devnull', - stdout = 'devnull', - stderr = 'devnull', - }) - - local attempts = 0 - -- Wait for replica to connect. - while box.info.replication[3] == nil or - box.info.replication[3].downstream.status ~= 'follow' do - - attempts = attempts + 1 - if attempts == 30 then - error('wait for replica connection') - end - fiber.sleep(0.1) - end - - local conn = require('net.box').connect('127.0.0.1:3396') - - conn:eval([[ - box.cfg{ - replication = { - 'replicator:password@127.0.0.1:3399', - 'replicator:password@127.0.0.1:3398', - 'replicator:password@127.0.0.1:3396', - }, - listen = '127.0.0.1:3397', - replication_connect_quorum = 4, - } - ]]) - - conn:eval('rawset(_G, "queue", require("queue"))') - - test:is(conn:call('queue.state'), 'INIT', 'check queue state') - test:is(conn:call('box.info').ro, true, 'check read only') - test:is(conn:call('box.info').ro_reason, 'orphan', 'check ro reason') - - conn:eval('box.cfg{replication_connect_quorum = 2}') - - local attempts = 0 - while conn:call('queue.state') ~= 'RUNNING' and attempts < 50 do - fiber.sleep(0.1) - attempts = attempts + 1 - end - test:is(conn:call('queue.state'), 'RUNNING', 'check queue state after orphan') -end) - -rawset(_G, 'queue', nil) -tnt.finish() -os.exit(test:check() and 0 or 1) --- vim: set ft=lua : diff --git a/t/benchmark/async_python.lua b/t/benchmark/async_python.lua deleted file mode 100755 index c084c74a..00000000 --- a/t/benchmark/async_python.lua +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env tarantool - -box.cfg{listen = 3301, wal_mode='none'} -box.schema.user.grant('guest', 'read,write,execute', 'universe') - -queue = require('queue') -local fiber = require('fiber') - -queue.create_tube('mail_msg', 'fifottl') - -local producer = fiber.create(function() - while true do - queue.tube.mail_msg:put('1') - fiber.sleep(0.2) - end -end) diff --git a/t/benchmark/async_python.py b/t/benchmark/async_python.py deleted file mode 100644 index c6178548..00000000 --- a/t/benchmark/async_python.py +++ /dev/null @@ -1,58 +0,0 @@ -import psutil - -from hurry.filesize import size, alternative - -import gevent -import gtarantool - -def worker(tnt, timeout): - while True: - try: - task = tnt.call("queue.tube.mail_msg:take", (timeout, )) - if task: - tnt.call("queue.tube.mail_msg:ack", (task[0][0], )) - except Exception as e: - print "Worker connection is closed", e - return - -def spawn_workers(workers, timeout, connection): - jobs = [] - for _ in range(workers): - jobs.append(gevent.spawn(worker, connection, timeout)) - return - -def find_pid_benchmark(): - for proc in psutil.process_iter(): - if proc.name().find('tarantool') >= 0: - return proc - return None - -def spawn_printer(): - proc = find_pid_benchmark() - if proc == None: - raise Exception("Can't find benchmark process") - tnt = gtarantool.connect("127.0.0.1", 3301) - while True: - try: - stat = tnt.call("box.stat")[0][0] - except: - print "Printer connection is closed" - return - print "%6.2f CPU | %6s MEM [ DELETE %6d rps | INSERT %6d rps | SELECT %6d rps ]" % ( - proc.cpu_percent(), size(proc.memory_info().rss, system=alternative), - stat['DELETE']['rps'], - stat['INSERT']['rps'], - stat['SELECT']['rps'] - ) - gevent.sleep(1) - -def main(): - workers = 200 - timeout = 2.5 - wcon = gtarantool.connect("127.0.0.1", 3301) - spawn_workers(workers, timeout, wcon) - spawn_printer() - return 0 - -if __name__ == "__main__": - exit(main()) diff --git a/t/benchmark/busy_utubes.lua b/t/benchmark/busy_utubes.lua deleted file mode 100644 index 3e95c3ba..00000000 --- a/t/benchmark/busy_utubes.lua +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env tarantool - -local clock = require('clock') -local os = require('os') -local fiber = require('fiber') -local queue = require('queue') - --- Set the number of consumers. -local consumers_count = 10 --- Set the number of tasks processed by one consumer per iteration. -local batch_size = 150000 - -local barrier = fiber.cond() -local wait_count = 0 - -box.cfg() - -local test_queue = queue.create_tube('test_queue', 'utube', - {temporary = true, storage_mode = queue.driver.utube.STORAGE_MODE_READY_BUFFER}) - -local function prepare_tasks() - local test_data = 'test data' - - for i = 1, consumers_count do - for _ = 1, batch_size do - test_queue:put(test_data, {utube = tostring(i)}) - end - end -end - -local function prepare_consumers() - local consumers = {} - - for i = 1, consumers_count do - consumers[i] = fiber.create(function() - wait_count = wait_count + 1 - -- Wait for all consumers to start. - barrier:wait() - - -- Ack the tasks. - for _ = 1, batch_size do - local task = test_queue:take() - fiber.yield() - test_queue:ack(task[1]) - end - - wait_count = wait_count + 1 - end) - end - - return consumers -end - -local function multi_consumer_bench() - --- Wait for all consumer fibers. - local wait_all = function() - while (wait_count ~= consumers_count) do - fiber.yield() - end - wait_count = 0 - end - - fiber.set_max_slice(100) - - prepare_tasks() - - -- Wait for all consumers to start. - local consumers = prepare_consumers() - wait_all() - - -- Start timing of task confirmation. - local start_ack_time = clock.proc64() - barrier:broadcast() - -- Wait for all tasks to be acked. - wait_all() - -- Complete the timing of task confirmation. - local complete_time = clock.proc64() - - -- Print the result in milliseconds. - print(string.format("Time it takes to confirm the tasks: %i ms", - tonumber((complete_time - start_ack_time) / 10^6))) -end - --- Start benchmark. -multi_consumer_bench() - --- Cleanup. -test_queue:drop() - -os.exit(0) diff --git a/t/benchmark/many_utubes.lua b/t/benchmark/many_utubes.lua deleted file mode 100644 index 63a08b3d..00000000 --- a/t/benchmark/many_utubes.lua +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env tarantool - -local clock = require('clock') -local os = require('os') -local fiber = require('fiber') -local queue = require('queue') - --- Set the number of consumers. -local consumers_count = 30000 - -local barrier = fiber.cond() -local wait_count = 0 - -box.cfg() - -local test_queue = queue.create_tube('test_queue', 'utube', - {temporary = true, storage_mode = queue.driver.utube.STORAGE_MODE_READY_BUFFER}) - -local function prepare_tasks() - local test_data = 'test data' - - for i = 1, consumers_count do - test_queue:put(test_data, {utube = tostring(i)}) - end -end - -local function prepare_consumers() - local consumers = {} - - for i = 1, consumers_count do - consumers[i] = fiber.create(function() - wait_count = wait_count + 1 - -- Wait for all consumers to start. - barrier:wait() - - -- Ack the task. - local task = test_queue:take() - test_queue:ack(task[1]) - - wait_count = wait_count + 1 - end) - end - - return consumers -end - -local function multi_consumer_bench() - --- Wait for all consumer fibers. - local wait_all = function() - while (wait_count ~= consumers_count) do - fiber.yield() - end - wait_count = 0 - end - - fiber.set_max_slice(100) - - -- Wait for all consumers to start. - local consumers = prepare_consumers() - wait_all() - - -- Start timing creation of tasks. - local start_put_time = clock.proc64() - prepare_tasks() - -- Start timing of task confirmation. - local start_ack_time = clock.proc64() - barrier:broadcast() - -- Wait for all tasks to be acked. - wait_all() - -- Complete the timing of task confirmation. - local complete_time = clock.proc64() - - -- Print results in milliseconds. - print(string.format("Time it takes to fill the queue: %i ms", - tonumber((start_ack_time - start_put_time) / 10^6))) - print(string.format("Time it takes to confirm the tasks: %i ms", - tonumber((complete_time - start_ack_time) / 10^6))) -end - --- Start benchmark. -multi_consumer_bench() - --- Cleanup. -test_queue:drop() - -os.exit(0) diff --git a/t/benchmark/multi_consumer_work.lua b/t/benchmark/multi_consumer_work.lua deleted file mode 100644 index 361909df..00000000 --- a/t/benchmark/multi_consumer_work.lua +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env tarantool - -local clock = require('clock') -local os = require('os') -local fiber = require('fiber') -local queue = require('queue') - --- Set the number of consumers. -local consumers_count = 100 --- Set the number of tasks processed by one consumer per iteration. -local batch_size = 100 - -local barrier = fiber.cond() -local wait_count = 0 - -box.cfg() - -local test_queue = queue.create_tube('test_queue', 'fifo', {temporary = true}) - -local function prepare_consumers() - local consumers = {} - local test_data = 'test data' - - for i = 1, consumers_count do - consumers[i] = fiber.create(function() - wait_count = wait_count + 1 - -- Wait for all consumers to start. - barrier:wait() - - -- Create test tasks. - for j = 1, batch_size do - test_queue:put(test_data) - end - - wait_count = wait_count + 1 - -- Wait for all consumers to create tasks. - barrier:wait() - - -- Ack the tasks. - for j = 1, batch_size do - local task = test_queue:take() - test_queue:ack(task[1]) - end - - wait_count = wait_count + 1 - end) - end - - return consumers -end - -local function multi_consumer_bench() - --- Wait for all consumer fibers. - local wait_all = function() - while (wait_count ~= consumers_count) do - fiber.yield() - end - wait_count = 0 - end - - -- Wait for all consumers to start. - local consumers = prepare_consumers() - wait_all() - - -- Start timing creation of tasks. - local start_put_time = clock.proc64() - barrier:broadcast() - -- Wait for all consumers to create tasks. - wait_all() - -- Complete timing creation of tasks. - local start_ack_time = clock.proc64() - barrier:broadcast() - -- Wait for all tasks to be acked. - wait_all() - -- Complete the timing of task confirmation. - local complete_time = clock.proc64() - - --Print the result in microseconds - print(string.format("Time it takes to fill the queue: %i", - tonumber((start_ack_time - start_put_time) / 10^3))) - print(string.format("Time it takes to confirm the tasks: %i", - tonumber((complete_time - start_ack_time) / 10^3))) -end - --- Start benchmark. -multi_consumer_bench() - --- Cleanup. -test_queue:drop() - -os.exit(0) diff --git a/t/tnt/init.lua b/t/tnt/init.lua deleted file mode 100644 index 94442523..00000000 --- a/t/tnt/init.lua +++ /dev/null @@ -1,240 +0,0 @@ -local fio = require('fio') -local log = require('log') -local yaml = require('yaml') -local errno = require('errno') -local fiber = require('fiber') -local netbox = require('net.box') - -local dir = os.getenv('QUEUE_TMP') -local cleanup = false - -local qc = require('queue.compat') -local vinyl_name = qc.vinyl_name -local snapdir_optname = qc.snapdir_optname -local logger_optname = qc.logger_optname - -local bind_master = os.getenv('QUEUE_MASTER_ADDR') -local bind_replica = os.getenv('QUEUE_REPLICA_ADDR') -local dir_replica = nil -local replica = nil - -if bind_master == nil then - bind_master = '127.0.0.1:3398' -end - -if bind_replica == nil then - bind_replica = '127.0.0.1:3399' -end - -if dir == nil then - dir = fio.tempdir() - cleanup = true -end - -local function tnt_prepare(cfg_args) - cfg_args = cfg_args or {} - local files = fio.glob(fio.pathjoin(dir, '*')) - for _, file in pairs(files) do - if fio.basename(file) ~= 'tarantool.log' then - log.info("skip removing %s", file) - fio.unlink(file) - end - end - - cfg_args['wal_dir'] = dir - cfg_args[snapdir_optname()] = dir - cfg_args[logger_optname()] = fio.pathjoin(dir, 'tarantool.log') - if vinyl_name() then - local vinyl_optname = vinyl_name() .. '_dir' - cfg_args[vinyl_optname] = dir - end - - box.cfg(cfg_args) -end - --- Creates master and replica setup for queue states switching tests. -local function tnt_cluster_prepare(cfg_args) - -- Since version 2.4.1, Tarantool has the popen built-in module - -- that supports execution of external programs. - if not qc.check_version({2, 4, 1}) then - error('this test requires tarantool >= 2.4.1') - return false - end - - -- Prepare master. - cfg_args = cfg_args or {} - local files = fio.glob(fio.pathjoin(dir, '*')) - for _, file in pairs(files) do - if fio.basename(file) ~= 'tarantool.log' then - log.info("skip removing %s", file) - fio.unlink(file) - end - end - - cfg_args['wal_dir'] = dir - cfg_args['read_only'] = false - cfg_args[snapdir_optname()] = dir - cfg_args[logger_optname()] = fio.pathjoin(dir, 'tarantool.log') - cfg_args['listen'] = bind_master - cfg_args['replication'] = {'replicator:password@' .. bind_master} - if vinyl_name() then - local vinyl_optname = vinyl_name() .. '_dir' - cfg_args[vinyl_optname] = dir - end - - box.cfg(cfg_args) - -- Allow guest all operations. - box.schema.user.grant('guest', 'read, write, execute, create, drop', 'universe') - box.schema.user.create('replicator', {password = 'password'}) - box.schema.user.grant('replicator', 'replication') - - -- Prepare replica. - dir_replica = fio.tempdir() - - local vinyl_opt = nil - if vinyl_name() then - vinyl_opt = ', ' .. vinyl_name() .. '_dir = \'' .. dir_replica .. '\'' - else - vinyl_opt = '' - end - - local cmd_replica = { - arg[-1], - '-e', - [[ - box.cfg { - read_only = true, - replication = 'replicator:password@]] .. bind_master .. - '\', listen = \'' .. bind_replica .. - '\', wal_dir = \'' .. dir_replica .. - '\', ' .. snapdir_optname() .. ' = \'' .. dir_replica .. - '\', ' .. logger_optname() .. ' = \'' .. - fio.pathjoin(dir_replica, 'tarantool.log') .. '\'' .. - vinyl_opt .. - '}' - } - - replica = require('popen').new(cmd_replica, { - stdin = 'devnull', - stdout = 'devnull', - stderr = 'devnull', - }) - - local attempts = 0 - - -- Wait for replica to connect. - while box.info.replication[2] == nil or box.info.replication[2].downstream.status ~= 'follow' do - attempts = attempts + 1 - if attempts == 30 then - error('wait for replica connection') - end - fiber.sleep(0.1) - end - - box.cfg({replication = - {'replicator:password@' .. bind_replica, - 'replicator:password@' .. bind_master - } - }) - - -- Wait for connect to replica. - attempts = 0 - - while box.info.replication[2].upstream.status ~= 'follow' do - attempts = attempts + 1 - if attempts == 30 then - error('wait for replica failed') - end - fiber.sleep(0.1) - end -end - -local function connect_replica() - if not replica then - return nil - end - - return netbox.connect(bind_replica) -end - -local function connect_master() - return netbox.connect(bind_master) -end - --- Wait for replica to connect. -local function wait_replica() - local attempts = 0 - - while true do - if #box.info.replication == 2 then - return true - end - attempts = attempts + 1 - if attempts == 10 then - return false - end - fiber.sleep(0.1) - end - - return false -end - -return { - finish = function(code) - local files = fio.glob(fio.pathjoin(dir, '*')) - for _, file in pairs(files) do - if fio.basename(file) == 'tarantool.log' and not cleanup then - log.info("skip removing %s", file) - else - log.info("remove %s", file) - fio.unlink(file) - end - end - if cleanup then - log.info("rmdir %s", dir) - fio.rmdir(dir) - end - if dir_replica then - local files = fio.glob(fio.pathjoin(dir, '*')) - for _, file in pairs(files) do - log.info("remove %s", file) - fio.unlink(file) - end - end - if replica then - replica:kill() - replica:wait() - end - end, - - dir = function() - return dir - end, - - cleanup = function() - return cleanup - end, - - logfile = function() - return fio.pathjoin(dir, 'tarantool.log') - end, - - log = function() - local fh = fio.open(fio.pathjoin(dir, 'tarantool.log'), 'O_RDONLY') - if fh == nil then - box.error(box.error.PROC_LUA, errno.strerror()) - end - - local data = fh:read(16384) - fh:close() - return data - end, - - cfg = tnt_prepare, - cluster = { - cfg = tnt_cluster_prepare, - wait_replica = wait_replica, - connect_replica = connect_replica, - connect_master = connect_master - } -} diff --git a/tarantool.cfg b/tarantool.cfg new file mode 100644 index 00000000..debef0e5 --- /dev/null +++ b/tarantool.cfg @@ -0,0 +1,78 @@ +readahead = 16384 + +primary_port = 33020 +secondary_port = 33021 +admin_port = 33022 + + +space = [ + { + enabled = 1, + index = [ + { + type = "TREE", + unique = 1, + key_field = [ + { + fieldno = 0, + type = "STR" + } + ] + }, + { + type = "TREE", + unique = 0, + key_field = [ + { + fieldno = 1, # tube + type = "STR" + }, + { + fieldno = 2, # status + type = "STR" + }, + { + fieldno = 4, # ipri + type = "STR" + }, + { + fieldno = 5 # pri + type = "STR" + } + ] + }, + { + type = "TREE", + unique = 0, + key_field = [ + { + fieldno = 1, # tube + type = "STR" + }, + { + fieldno = 3, # next_event + type = "NUM64" + } + ] + }, +# { +# type = "TREE", +# unique = 0, +# key_field = [ +# { +# fieldno = 1, # tube +# type = "STR" +# }, +# { +# fieldno = 2, # status +# type = "STR" +# }, +# { +# fieldno = 12, # task data +# type = "STR" +# } +# ] +# } + ] + } +]