Add sharding (#17)

config/database.php

@@ -8,5 +8,17 @@
             'port' => env('REDIS_PORT', 6379),
             'database' => 0,
         ],
+        'qless1' => [
+            'host' => env('REDIS_HOST', '127.0.0.1'),
+            'password' => env('REDIS_PASSWORD', null),
+            'port' => env('REDIS_PORT', 6379),
+            'database' => 1,
+        ],
+        'qless2' => [
+            'host' => env('REDIS_HOST', '127.0.0.1'),
+            'password' => env('REDIS_PASSWORD', null),
+            'port' => env('REDIS_PORT', 6379),
+            'database' => 2,
+        ],
     ],
 ];

config/queue.php

@@ -8,7 +8,8 @@
             'driver' => 'qless',
             'connection' => 'qless',
             'queue' => 'default',
-            'redis_connection' => 'qless',
+//            'redis_connection' => 'qless',
+            'redis_connection' => ['qless1', 'qless2'],
         ],
     ],
 ];

src/Job/AbstractJob.php

@@ -6,7 +6,7 @@
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Contracts\Support\Arrayable;
 use Illuminate\Foundation\Bus\Dispatchable;
-use LaravelQless\Queue\QlessQueue;
+use LaravelQless\Queue\QlessConnector;
 use Qless\Jobs\BaseJob;
 use LaravelQless\Contracts\QlessJob;
 
@@ -63,16 +63,14 @@ protected function completeSync(): self
 
     private function completeImmediately(): void
     {
-        /**@var QlessQueue $queue  */
-        $queue = app(QlessQueue::class, [
-            'config' => [
-                'queue' => $this->queue,
-                'connection' => $this->connection
-            ]
+        $connector = new QlessConnector();
+        $queue = $connector->connect([
+            'queue' => $this->queue,
+            'connection' => $this->connection
         ]);
 
         $jid = $queue->push($this, $this->data, $this->queue);
-        $connection = $queue->getConnection()->queues[$this->queue];
+        $connection = $queue->getCurrentConnection()->queues[$this->queue];
 
         /**@var BaseJob $job */
         $job = $connection->popByJid($jid);

src/Queue/QlessConnectionHandler.php

@@ -0,0 +1,77 @@
+<?php
+
+namespace LaravelQless\Queue;
+
+use ArrayIterator;
+use Qless\Client;
+
+/**
+ * Class QlessConnectionHandler
+ * @package LaravelQless\Queue
+ */
+class QlessConnectionHandler
+{
+
+    /** @var Client[] */
+    private $clients;
+
+    /** @var ArrayIterator */
+    private $clientIterator;
+
+    /**
+     * QlessConnectionHandler constructor.
+     * @param Client[] $clients
+     */
+    public function __construct(array $clients)
+    {
+        $this->init($clients);
+    }
+
+    private function init(array $clients): void
+    {
+        foreach ($clients as $client) {
+            if (!$client instanceof Client) {
+                continue;
+            }
+            $this->clients[] = $client;
+
+        }
+        if (empty($this->clients)) {
+            throw new \Exception("No configs found");
+        }
+
+        $this->clientIterator = new ArrayIterator($this->clients);
+    }
+
+    public function getRandomClient(): Client
+    {
+        return $this->clients[array_rand($this->clients)];
+    }
+
+    public function getAllClients(): array
+    {
+        return $this->clients;
+    }
+
+    public function getCurrentClient(): Client
+    {
+        if ($this->clientIterator->current() === null) {
+            return $this->getNextClient();
+        }
+
+        return $this->clientIterator->current();
+    }
+
+    public function getNextClient(): Client
+    {
+        if ($this->clientIterator->current() === null) {
+            $this->clientIterator->rewind();
+        }
+
+        $currentClient = $this->clientIterator->current();
+        $this->clientIterator->next();
+
+        return $currentClient;
+    }
+
+}

src/Queue/QlessConnector.php

@@ -13,22 +13,35 @@
  */
 class QlessConnector implements ConnectorInterface
 {
+    private const REDIS_CONNECTION_CONFIG_KEY = 'redis_connection';
+    private const CONFIG_PATH_PREFIX = 'database.redis.';
+
+    private const DEFAULT_CONNECTION_CONFIG = 'qless';
+
     /**
-    * Establish a queue connection.
-    *
-    * @param array $config
-    *
-    * @return QlessQueue
-    */
+     * Establish a queue connection.
+     *
+     * @param array $config
+     *
+     * @return QlessQueue
+     */
     public function connect(array $config): QlessQueue
     {
-         $redisConnection = Arr::get($config, 'redis_connection', 'qless');
+        $redisConnection = Arr::get($config, self::REDIS_CONNECTION_CONFIG_KEY, self::DEFAULT_CONNECTION_CONFIG);
+
+        if (!is_array($redisConnection)) {
+            $redisConnection = [$redisConnection];
+        }
 
-         $redisConfig = Config::get('database.redis.' . $redisConnection, []);
+        $clients = [];
+        foreach ($redisConnection as $connection) {
+            $qlessConfig = Config::get(self::CONFIG_PATH_PREFIX . $connection, []);
+            $clients[] = new Client($qlessConfig);
+        }
 
-         return new QlessQueue(
-             new Client($redisConfig),
-             $config
-         );
+        return new QlessQueue(
+            new QlessConnectionHandler($clients),
+            $config
+        );
     }
 }

src/Queue/QlessQueue.php

@@ -22,11 +22,6 @@ class QlessQueue extends Queue implements QueueContract
 
     private const WORKER_PREFIX = 'laravel_';
 
-    /**
-     * @var Client
-     */
-    private $connect;
-
     /**
      * @var string
      */
@@ -37,14 +32,17 @@ class QlessQueue extends Queue implements QueueContract
      */
     private $config;
 
+    /** @var QlessConnectionHandler */
+    private $clients;
+
     /**
      * QlessQueue constructor.
-     * @param Client $connect
+     * @param QlessConnectionHandler $clients
      * @param array $config
      */
-    public function __construct(Client $connect, array $config)
+    public function __construct(QlessConnectionHandler $clients, array $config)
     {
-        $this->connect = $connect;
+        $this->clients = $clients;
         $this->defaultQueue = $config['queue'] ?? null;
         $this->connectionName = $config['connection'] ?? '';
         $this->config = $config;
@@ -53,20 +51,20 @@ public function __construct(Client $connect, array $config)
     /**
      * Get the size of the queue.
      *
-     * @param  string  $queue
+     * @param string $queue
      * @return int
      */
     public function size($queue = null): int
     {
-        return $this->getConnection()->length($queue ?? '');
+        return $this->getNextConnection()->length($queue ?? '');
     }
 
     /**
      * Push a raw payload onto the queue.
      *
-     * @param  string  $payload
-     * @param  string  $queueName
-     * @param  array   $options
+     * @param string $payload
+     * @param string $queueName
+     * @param array $options
      * @return mixed
      */
     public function pushRaw($payload, $queueName = null, array $options = [])
@@ -75,7 +73,7 @@ public function pushRaw($payload, $queueName = null, array $options = [])
 
         $queueName = $queueName ?? $this->defaultQueue;
 
-        $queue = $this->getConnection()->queues[$queueName];
+        $queue = $this->getRandomConnection()->queues[$queueName];
 
         $qlessOptions = $payloadData['data'][self::JOB_OPTIONS_KEY] ?? [];
 
@@ -96,23 +94,23 @@ public function pushRaw($payload, $queueName = null, array $options = [])
     /**
      * Push a new job onto the queue.
      *
-     * @param  string|object  $job
-     * @param  mixed   $data
-     * @param  string  $queueName
+     * @param string|object $job
+     * @param mixed $data
+     * @param string $queueName
      * @return mixed
      */
     public function push($job, $data = '', $queueName = null)
     {
-        return $this->pushRaw($this->makePayload($job, (array) $data), $queueName);
+        return $this->pushRaw($this->makePayload($job, (array)$data), $queueName);
     }
 
     /**
      * Push a new job onto the queue after a delay.
      *
-     * @param  \DateTimeInterface|\DateInterval|int  $delay
-     * @param  string|object  $job
-     * @param  mixed   $data
-     * @param  string  $queueName
+     * @param \DateTimeInterface|\DateInterval|int $delay
+     * @param string|object $job
+     * @param mixed $data
+     * @param string $queueName
      * @return mixed
      */
     public function later($delay, $job, $data = '', $queueName = null)
@@ -139,7 +137,7 @@ public function later($delay, $job, $data = '', $queueName = null)
     public function recur(int $interval, string $job, array $data, ?string $queueName = null): string
     {
         /** @var \Qless\Queues\Queue $queue */
-        $queue = $this->getConnection()->queues[$queueName];
+        $queue = $this->getNextConnection()->queues[$queueName];
 
         $options = $data[self::JOB_OPTIONS_KEY] ?? [];
         $options = array_merge($options, ['interval' => $interval]);
@@ -160,16 +158,18 @@ public function recur(int $interval, string $job, array $data, ?string $queueNam
     /**
      * Pop the next job off of the queue.
      *
-     * @param  string  $queueName
+     * @param string $queueName
      * @return QlessJob|null
      */
     public function pop($queueName = null)
     {
+        $connection = $this->getNextConnection();
+
         /** @var \Qless\Queues\Queue $queue */
-        $queue = $this->getConnection()->queues[$queueName];
+        $queue = $connection->queues[$queueName];
 
         /** @var BaseJob $job */
-        $job = $queue->pop(self::WORKER_PREFIX . $this->connect->getWorkerName());
+        $job = $queue->pop(self::WORKER_PREFIX . $connection->getWorkerName());
 
         if (!$job) {
             return null;
@@ -195,10 +195,14 @@ public function subscribe(string $topic, string $queueName = null): bool
     {
         $queueName = $queueName ?? $this->defaultQueue;
 
-        /** @var \Qless\Queues\Queue $queue */
-        $queue = $this->getConnection()->queues[$queueName];
+        $result = true;
+        foreach ($this->getAllConnections() as $connection) {
+            /** @var \Qless\Queues\Queue $queue */
+            $queue = $connection->queues[$queueName];
+            $result = $queue->subscribe($topic) && $result;
+        }
 
-        return $queue->subscribe($topic);
+        return $result;
     }
 
     /**
@@ -210,10 +214,14 @@ public function unSubscribe(string $topic, string $queueName = null): bool
     {
         $queueName = $queueName ?? $this->defaultQueue;
 
-        /** @var \Qless\Queues\Queue $queue */
-        $queue = $this->getConnection()->queues[$queueName];
+        $result = true;
+        foreach ($this->getAllConnections() as $connection) {
+            /** @var \Qless\Queues\Queue $queue */
+            $queue = $connection->queues[$queueName];
+            $result = $queue->unSubscribe($topic) && $result;
+        }
 
-        return $queue->unSubscribe($topic);
+        return $result;
     }
 
     /**
@@ -225,7 +233,7 @@ public function unSubscribe(string $topic, string $queueName = null): bool
      */
     public function pushToTopic(string $topicName, string $job, array $data = [], array $options = [])
     {
-        $topic = new Topic($topicName, $this->getConnection());
+        $topic = new Topic($topicName, $this->getRandomConnection());
 
         $qlessOptions = $payloadData['data'][self::JOB_OPTIONS_KEY] ?? [];
         $options = array_merge($qlessOptions, $options);
@@ -281,10 +289,25 @@ protected function makePayload($job, $data = [], $options = []): string
     }
 
     /**
-     * @return Client
+     * @return Client[]
      */
-    public function getConnection(): Client
+    public function getAllConnections(): array
+    {
+        return $this->clients->getAllClients();
+    }
+
+    public function getRandomConnection(): Client
+    {
+        return $this->clients->getNextClient();
+    }
+
+    public function getNextConnection(): Client
+    {
+        return $this->clients->getNextClient();
+    }
+
+    public function getCurrentConnection(): Client
     {
-        return $this->connect;
+        return $this->clients->getCurrentClient();
     }
 }