Skip to content

Commit a998870

Browse files
duhonduhon
authored
Merge pull request #2888 from magento-performance/MAGETWO-93669
Tasks: - MAGETWO-93669 [Forwardport] Implement parallelization for Category Product Indexer - MAGETWO-93671 [Forwardport] Implement parallelization for Search Indexer
2 parents d120ac1 + 4089205 commit a998870

File tree

11 files changed

+296
-26
lines changed

11 files changed

+296
-26
lines changed

‎app/code/Magento/Catalog/Model/Indexer/Category/Product/Action/Full.php‎

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
use Magento\Catalog\Model\ResourceModel\Indexer\ActiveTableSwitcher;
99
use Magento\Framework\DB\Query\Generator as QueryGenerator;
1010
use Magento\Framework\App\ResourceConnection;
11+
use Magento\Indexer\Model\ProcessManager;
1112

1213
/**
1314
* Class Full reindex action
@@ -44,6 +45,11 @@ class Full extends \Magento\Catalog\Model\Indexer\Category\Product\AbstractActio
4445
*/
4546
private $activeTableSwitcher;
4647

48+
/**
49+
* @var ProcessManager
50+
*/
51+
private $processManager;
52+
4753
/**
4854
* @param ResourceConnection $resource
4955
* @param \Magento\Store\Model\StoreManagerInterface $storeManager
@@ -52,9 +58,10 @@ class Full extends \Magento\Catalog\Model\Indexer\Category\Product\AbstractActio
5258
* @param \Magento\Framework\Indexer\BatchSizeManagementInterface|null $batchSizeManagement
5359
* @param \Magento\Framework\Indexer\BatchProviderInterface|null $batchProvider
5460
* @param \Magento\Framework\EntityManager\MetadataPool|null $metadataPool
55-
* @param \Magento\Indexer\Model\Indexer\StateFactory|null $stateFactory
5661
* @param int|null $batchRowsCount
5762
* @param ActiveTableSwitcher|null $activeTableSwitcher
63+
* @param ProcessManager $processManager
64+
* @SuppressWarnings(PHPMD.ExcessiveParameterList)
5865
*/
5966
public function __construct(
6067
\Magento\Framework\App\ResourceConnection $resource,
@@ -65,7 +72,8 @@ public function __construct(
6572
\Magento\Framework\Indexer\BatchProviderInterface $batchProvider = null,
6673
\Magento\Framework\EntityManager\MetadataPool $metadataPool = null,
6774
$batchRowsCount = null,
68-
ActiveTableSwitcher $activeTableSwitcher = null
75+
ActiveTableSwitcher $activeTableSwitcher = null,
76+
ProcessManager $processManager = null
6977
) {
7078
parent::__construct(
7179
$resource,
@@ -85,6 +93,7 @@ public function __construct(
8593
);
8694
$this->batchRowsCount = $batchRowsCount;
8795
$this->activeTableSwitcher = $activeTableSwitcher ?: $objectManager->get(ActiveTableSwitcher::class);
96+
$this->processManager = $processManager ?: $objectManager->get(ProcessManager::class);
8897
}
8998

9099
/**
@@ -133,6 +142,38 @@ public function execute()
133142
return $this;
134143
}
135144

145+
/**
146+
* Run reindexation
147+
*
148+
* @return void
149+
*/
150+
protected function reindex()
151+
{
152+
$userFunctions = [];
153+
154+
foreach ($this->storeManager->getStores() as $store) {
155+
if ($this->getPathFromCategoryId($store->getRootCategoryId())) {
156+
$userFunctions[$store->getId()] = function () use ($store) {
157+
return $this->reindexStore($store);
158+
};
159+
}
160+
}
161+
162+
$this->processManager->execute($userFunctions);
163+
}
164+
165+
/**
166+
* Execute indexation by store
167+
*
168+
* @param \Magento\Store\Model\Store $store
169+
*/
170+
private function reindexStore($store)
171+
{
172+
$this->reindexRootCategory($store);
173+
$this->reindexAnchorCategories($store);
174+
$this->reindexNonAnchorCategories($store);
175+
}
176+
136177
/**
137178
* Publish data from tmp to replica table
138179
*

‎app/code/Magento/CatalogSearch/Model/Indexer/Fulltext.php‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,13 @@
1010
use Magento\CatalogSearch\Model\ResourceModel\Fulltext as FulltextResource;
1111
use Magento\Framework\Indexer\Dimension\DimensionProviderInterface;
1212
use Magento\Store\Model\StoreDimensionProvider;
13+
use Magento\Indexer\Model\ProcessManager;
1314

1415
/**
1516
* Provide functionality for Fulltext Search indexing.
1617
*
18+
* @SuppressWarnings(PHPMD.CouplingBetweenObjects)
19+
*
1720
* @api
1821
* @since 100.0.2
1922
*/
@@ -62,14 +65,20 @@ class Fulltext implements
6265
*/
6366
private $dimensionProvider;
6467

68+
/**
69+
* @var ProcessManager
70+
*/
71+
private $processManager;
72+
6573
/**
6674
* @param FullFactory $fullActionFactory
6775
* @param IndexerHandlerFactory $indexerHandlerFactory
6876
* @param FulltextResource $fulltextResource
69-
* @param array $data
7077
* @param IndexSwitcherInterface $indexSwitcher
7178
* @param StateFactory $indexScopeStateFactory
7279
* @param DimensionProviderInterface $dimensionProvider
80+
* @param array $data
81+
* @param ProcessManager $processManager
7382
*/
7483
public function __construct(
7584
FullFactory $fullActionFactory,
@@ -78,7 +87,8 @@ public function __construct(
7887
IndexSwitcherInterface $indexSwitcher,
7988
StateFactory $indexScopeStateFactory,
8089
DimensionProviderInterface $dimensionProvider,
81-
array $data
90+
array $data,
91+
ProcessManager $processManager = null
8292
) {
8393
$this->fullAction = $fullActionFactory->create(['data' => $data]);
8494
$this->indexerHandlerFactory = $indexerHandlerFactory;
@@ -87,6 +97,9 @@ public function __construct(
8797
$this->indexSwitcher = $indexSwitcher;
8898
$this->indexScopeState = $indexScopeStateFactory->create();
8999
$this->dimensionProvider = $dimensionProvider;
100+
$this->processManager = $processManager ?: \Magento\Framework\App\ObjectManager::getInstance()->get(
101+
ProcessManager::class
102+
);
90103
}
91104

92105
/**
@@ -145,9 +158,13 @@ public function executeByDimension(array $dimensions, \Traversable $entityIds =
145158
*/
146159
public function executeFull()
147160
{
161+
$userFunctions = [];
148162
foreach ($this->dimensionProvider->getIterator() as $dimension) {
149-
$this->executeByDimension($dimension);
163+
$userFunctions[] = function () use ($dimension) {
164+
$this->executeByDimension($dimension);
165+
};
150166
}
167+
$this->processManager->execute($userFunctions);
151168
}
152169

153170
/**

‎app/code/Magento/CatalogSearch/Test/Unit/Model/Indexer/FulltextTest.php‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,11 @@ class FulltextTest extends \PHPUnit\Framework\TestCase
4444
*/
4545
private $dimensionProviderMock;
4646

47+
/**
48+
* @var \Magento\Indexer\Model\ProcessManager|\PHPUnit_Framework_MockObject_MockObject
49+
*/
50+
private $processManager;
51+
4752
protected function setUp()
4853
{
4954
$this->fullAction = $this->getClassMock(\Magento\CatalogSearch\Model\Indexer\Fulltext\Action\Full::class);
@@ -70,6 +75,11 @@ protected function setUp()
7075
$stateMock = $this->getMockBuilder(\Magento\CatalogSearch\Model\Indexer\Scope\State::class)
7176
->getMock();
7277
$objectManagerHelper = new ObjectManagerHelper($this);
78+
79+
$this->processManager = new \Magento\Indexer\Model\ProcessManager(
80+
$this->getClassMock(\Magento\Framework\App\ResourceConnection::class)
81+
);
82+
7383
$this->model = $objectManagerHelper->getObject(
7484
\Magento\CatalogSearch\Model\Indexer\Fulltext::class,
7585
[
@@ -80,6 +90,7 @@ protected function setUp()
8090
'indexSwitcher' => $this->indexSwitcher,
8191
'dimensionProvider' => $this->dimensionProviderMock,
8292
'indexScopeState' => $stateMock,
93+
'processManager' => $this->processManager,
8394
]
8495
);
8596
}

‎app/code/Magento/CatalogSearch/composer.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
"magento/framework": "*",
1010
"magento/module-backend": "*",
1111
"magento/module-catalog": "*",
12+
"magento/module-indexer": "100.2.*",
1213
"magento/module-catalog-inventory": "*",
1314
"magento/module-customer": "*",
1415
"magento/module-directory": "*",

‎app/code/Magento/CatalogSearch/etc/module.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
<sequence>
1111
<module name="Magento_Search"/>
1212
<module name="Magento_Catalog"/>
13+
<module name="Magento_Indexer"/>
1314
<module name="Magento_CatalogRule" />
1415
<module name="Magento_CatalogInventory" />
1516
</sequence>

‎app/code/Magento/Elasticsearch/Model/Client/Elasticsearch.php‎

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ class Elasticsearch implements ClientInterface
1717
/**
1818
* Elasticsearch Client instance
1919
*
20-
* @var \Elasticsearch\Client
20+
* @var \Elasticsearch\Client[]
2121
*/
2222
protected $client;
2323

@@ -53,10 +53,19 @@ public function __construct(
5353
$config = $this->buildConfig($options);
5454
$elasticsearchClient = \Elasticsearch\ClientBuilder::fromConfig($config, true);
5555
}
56-
$this->client = $elasticsearchClient;
56+
$this->client[getmypid()] = $elasticsearchClient;
5757
$this->clientOptions = $options;
5858
}
5959

60+
private function getClient()
61+
{
62+
$pid = getmypid();
63+
if (!isset($this->client[$pid])) {
64+
$config = $this->buildConfig($this->clientOptions);
65+
$this->client[$pid] = \Elasticsearch\ClientBuilder::fromConfig($config, true);
66+
}
67+
return $this->client[$pid];
68+
}
6069
/**
6170
* Ping the Elasticsearch client
6271
*
@@ -65,7 +74,7 @@ public function __construct(
6574
public function ping()
6675
{
6776
if ($this->pingResult === null) {
68-
$this->pingResult = $this->client->ping(['client' => ['timeout' => $this->clientOptions['timeout']]]);
77+
$this->pingResult = $this->getClient()->ping(['client' => ['timeout' => $this->clientOptions['timeout']]]);
6978
}
7079
return $this->pingResult;
7180
}
@@ -110,7 +119,7 @@ private function buildConfig($options = [])
110119
*/
111120
public function bulkQuery($query)
112121
{
113-
$this->client->bulk($query);
122+
$this->getClient()->bulk($query);
114123
}
115124

116125
/**
@@ -122,7 +131,7 @@ public function bulkQuery($query)
122131
*/
123132
public function createIndex($index, $settings)
124133
{
125-
$this->client->indices()->create([
134+
$this->getClient()->indices()->create([
126135
'index' => $index,
127136
'body' => $settings,
128137
]);
@@ -136,7 +145,7 @@ public function createIndex($index, $settings)
136145
*/
137146
public function deleteIndex($index)
138147
{
139-
$this->client->indices()->delete(['index' => $index]);
148+
$this->getClient()->indices()->delete(['index' => $index]);
140149
}
141150

142151
/**
@@ -147,7 +156,7 @@ public function deleteIndex($index)
147156
*/
148157
public function isEmptyIndex($index)
149158
{
150-
$stats = $this->client->indices()->stats(['index' => $index, 'metric' => 'docs']);
159+
$stats = $this->getClient()->indices()->stats(['index' => $index, 'metric' => 'docs']);
151160
if ($stats['indices'][$index]['primaries']['docs']['count'] == 0) {
152161
return true;
153162
}
@@ -172,7 +181,7 @@ public function updateAlias($alias, $newIndex, $oldIndex = '')
172181
$params['body']['actions'][] = ['add' => ['alias' => $alias, 'index' => $newIndex]];
173182
}
174183

175-
$this->client->indices()->updateAliases($params);
184+
$this->getClient()->indices()->updateAliases($params);
176185
}
177186

178187
/**
@@ -183,7 +192,7 @@ public function updateAlias($alias, $newIndex, $oldIndex = '')
183192
*/
184193
public function indexExists($index)
185194
{
186-
return $this->client->indices()->exists(['index' => $index]);
195+
return $this->getClient()->indices()->exists(['index' => $index]);
187196
}
188197

189198
/**
@@ -198,7 +207,7 @@ public function existsAlias($alias, $index = '')
198207
if ($index) {
199208
$params['index'] = $index;
200209
}
201-
return $this->client->indices()->existsAlias($params);
210+
return $this->getClient()->indices()->existsAlias($params);
202211
}
203212

204213
/**
@@ -208,7 +217,7 @@ public function existsAlias($alias, $index = '')
208217
*/
209218
public function getAlias($alias)
210219
{
211-
return $this->client->indices()->getAlias(['name' => $alias]);
220+
return $this->getClient()->indices()->getAlias(['name' => $alias]);
212221
}
213222

214223
/**
@@ -267,7 +276,7 @@ public function addFieldsMapping(array $fields, $index, $entityType)
267276
foreach ($fields as $field => $fieldInfo) {
268277
$params['body'][$entityType]['properties'][$field] = $fieldInfo;
269278
}
270-
$this->client->indices()->putMapping($params);
279+
$this->getClient()->indices()->putMapping($params);
271280
}
272281

273282
/**
@@ -279,7 +288,7 @@ public function addFieldsMapping(array $fields, $index, $entityType)
279288
*/
280289
public function deleteMapping($index, $entityType)
281290
{
282-
$this->client->indices()->deleteMapping([
291+
$this->getClient()->indices()->deleteMapping([
283292
'index' => $index,
284293
'type' => $entityType,
285294
]);
@@ -293,8 +302,7 @@ public function deleteMapping($index, $entityType)
293302
*/
294303
public function query($query)
295304
{
296-
$params = array_merge($query, ['client' => ['timeout' => $this->clientOptions['timeout']]]);
297-
return $this->client->search($params);
305+
return $this->getClient()->search($query);
298306
}
299307

300308
/**
@@ -305,6 +313,6 @@ public function query($query)
305313
*/
306314
public function suggest($query)
307315
{
308-
return $this->client->suggest($query);
316+
return $this->getClient()->suggest($query);
309317
}
310318
}

0 commit comments

Comments
 (0)