Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

perf(sharing): Split the query to get the share and storage info into 2 #11553

Draft
wants to merge 2 commits into
base: perf/noid/move-item_type-validation-to-php
Choose a base branch
from
Draft

perf(sharing): Split the query to get the share and storage info into 2 #11553

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a6deb3e
perf(sharing): Move item_type validation to PHP
nickvergessen Feb 7, 2024
6657f04
perf(sharing): Split the query to get the share and storage info into 2
nickvergessen Feb 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
146 changes: 82 additions & 64 deletions lib/Share/RoomShareProvider.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,15 +74,15 @@ class RoomShareProvider implements IShareProvider {
private CappedMemoryCache $sharesByIdCache;

public function __construct(
private IDBConnection $dbConnection,
private ISecureRandom $secureRandom,
private IShareManager $shareManager,
private IEventDispatcher $dispatcher,
private Manager $manager,
private ParticipantService $participantService,
protected IDBConnection $dbConnection,
protected ISecureRandom $secureRandom,
protected IShareManager $shareManager,
protected IEventDispatcher $dispatcher,
protected Manager $manager,
protected ParticipantService $participantService,
protected ITimeFactory $timeFactory,
private IL10N $l,
private IMimeTypeLoader $mimeTypeLoader,
protected IL10N $l,
protected IMimeTypeLoader $mimeTypeLoader,
) {
$this->sharesByIdCache = new CappedMemoryCache();
}
Expand Down Expand Up @@ -682,6 +682,10 @@ public function getSharesByIds(array $ids, ?string $recipientId = null): array {
$shares = [];
while ($data = $cursor->fetch()) {
$id = $data['id'];
if ($data['item_type'] !== 'file' && $data['item_type'] !== 'folder') {
$this->sharesByIdCache[$id] = false;
continue;
}
if ($this->isAccessibleResult($data)) {
$share = $this->createShareObject($data);
$shares[] = $share;
Expand Down Expand Up @@ -739,15 +743,17 @@ private function resolveSharesForRecipient(array $shares, string $userId): array
$query = $qb->select('*')
->from('share')
->where($qb->expr()->in('parent', $qb->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY)))
->andWhere($qb->expr()->eq('share_with', $qb->createNamedParameter($userId)))
->andWhere($qb->expr()->orX(
$qb->expr()->eq('item_type', $qb->createNamedParameter('file')),
$qb->expr()->eq('item_type', $qb->createNamedParameter('folder'))
));
->andWhere($qb->expr()->eq('share_with', $qb->createNamedParameter($userId)));

$stmt = $query->executeQuery();

while ($data = $stmt->fetch()) {
if ($data['item_type'] !== 'file' && $data['item_type'] !== 'folder') {
continue;
}
if ($data['share_type'] !== self::SHARE_TYPE_USERROOM) {
continue;
}
$shareMap[$data['parent']]->setPermissions((int)$data['permissions']);
$shareMap[$data['parent']]->setTarget($data['file_target']);
}
Expand Down Expand Up @@ -799,63 +805,75 @@ public function getSharesByPath(Node $path): array {
public function getSharedWith($userId, $shareType, $node, $limit, $offset): array {
$allRooms = $this->manager->getRoomTokensForUser($userId);

/** @var IShare[] $shares */
$shares = [];
$query = $this->dbConnection->getQueryBuilder();
$query->select('s.*')
->from('share', 's')
->where($query->expr()->eq('s.share_type', $query->createNamedParameter(IShare::TYPE_ROOM)))
->andWhere($query->expr()->in('s.share_with', $query->createParameter('rooms')));

$start = 0;
while (true) {
$rooms = array_slice($allRooms, $start, 100);
$start += 100;
// Filter by node if provided
if ($node !== null) {
$query->andWhere($query->expr()->eq('s.file_source', $query->createNamedParameter($node->getId())));
}

if ($rooms === []) {
break;
}
if ($limit !== -1) {
$query->orderBy('s.id', 'ASC')
->setMaxResults($limit);
}

$qb = $this->dbConnection->getQueryBuilder();
$qb->select('s.*',
'f.fileid', 'f.path', 'f.permissions AS f_permissions', 'f.storage', 'f.path_hash',
'f.parent AS f_parent', 'f.name', 'f.mimetype', 'f.mimepart', 'f.size', 'f.mtime', 'f.storage_mtime',
'f.encrypted', 'f.unencrypted_size', 'f.etag', 'f.checksum'
)
->selectAlias('st.id', 'storage_string_id')
->from('share', 's')
->orderBy('s.id', 'ASC')
->leftJoin('s', 'filecache', 'f', $qb->expr()->eq('s.file_source', 'f.fileid'))
->leftJoin('f', 'storages', 'st', $qb->expr()->eq('f.storage', 'st.numeric_id'));

if ($limit !== -1) {
$qb->setMaxResults($limit);
}
/** @var array<int, array> $shareRows */
$shareRows = [];

// Filter by node if provided
if ($node !== null) {
$qb->andWhere($qb->expr()->eq('s.file_source', $qb->createNamedParameter($node->getId())));
}
/** @var array<int, ?array> $fileData */
$fileData = [];

foreach (array_chunk($allRooms, 1000) as $rooms) {
$query->setParameter('rooms', $rooms, IQueryBuilder::PARAM_STR_ARRAY);

$qb->andWhere($qb->expr()->eq('s.share_type', $qb->createNamedParameter(IShare::TYPE_ROOM)))
->andWhere($qb->expr()->in('s.share_with', $qb->createNamedParameter(
$rooms,
IQueryBuilder::PARAM_STR_ARRAY
)))
->andWhere($qb->expr()->orX(
$qb->expr()->eq('s.item_type', $qb->createNamedParameter('file')),
$qb->expr()->eq('s.item_type', $qb->createNamedParameter('folder'))
));

$cursor = $qb->executeQuery();
while ($data = $cursor->fetch()) {
if (!$this->isAccessibleResult($data)) {
$result = $query->executeQuery();
while ($row = $result->fetch()) {
if ($row['item_type'] !== 'file' && $row['item_type'] !== 'folder') {
continue;
}

if ($offset > 0) {
$offset--;
$shareRows[(int)$row['id']] = $row;
$fileData[(int)$row['file_source']] = null;
}
$result->closeCursor();
}

$queryFileCache = $this->dbConnection->getQueryBuilder();
$queryFileCache->select('f.fileid', 'f.path', 'f.permissions AS f_permissions', 'f.storage', 'f.path_hash',
'f.parent AS f_parent', 'f.name', 'f.mimetype', 'f.mimepart', 'f.size', 'f.mtime', 'f.storage_mtime',
'f.encrypted', 'f.unencrypted_size', 'f.etag', 'f.checksum')
->selectAlias('st.id', 'storage_string_id')
->from('filecache', 'f')
->leftJoin('f', 'storages', 'st', $queryFileCache->expr()->eq('f.storage', 'st.numeric_id'))
->where($queryFileCache->expr()->in('f.fileid', $queryFileCache->createParameter('fileIds')));

$allFileIds = array_keys($fileData);
foreach (array_chunk($allFileIds, 1000) as $fileIds) {
// Filecache and storage info
$queryFileCache->setParameter('fileIds', $fileIds, IQueryBuilder::PARAM_INT_ARRAY);

$result = $queryFileCache->executeQuery();
while ($row = $result->fetch()) {
if (!$this->isAccessibleResult($row)) {
continue;
}

$shares[] = $this->createShareObject($data);
$fileData[(int) $row['fileid']] = $row;
}
$cursor->closeCursor();
$result->closeCursor();
}

/** @var IShare[] $shares */
$shares = [];
foreach ($shareRows as $row) {
if (empty($fileData[(int)$row['file_source']])) {
continue;
}
$shares[] = $this->createShareObject(array_merge($row, $fileData[(int)$row['file_source']]));
}

$shares = $this->resolveSharesForRecipient($shares, $userId);
Expand Down Expand Up @@ -972,18 +990,18 @@ public function getAccessList($nodes, $currentAccess): array {
$types[] = self::SHARE_TYPE_USERROOM;
}

$qb->select('id', 'parent', 'share_type', 'share_with', 'file_source', 'file_target', 'permissions')
$qb->select('id', 'parent', 'share_type', 'share_with', 'file_source', 'file_target', 'permissions', 'item_type')
->from('share')
->where($qb->expr()->in('share_type', $qb->createNamedParameter($types, IQueryBuilder::PARAM_INT_ARRAY)))
->andWhere($qb->expr()->in('file_source', $qb->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY)))
->andWhere($qb->expr()->orX(
$qb->expr()->eq('item_type', $qb->createNamedParameter('file')),
$qb->expr()->eq('item_type', $qb->createNamedParameter('folder'))
));
->andWhere($qb->expr()->in('file_source', $qb->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY)));
$cursor = $qb->executeQuery();

$users = [];
while ($row = $cursor->fetch()) {
if ($row['item_type'] !== 'file' && $row['item_type'] !== 'folder') {
continue;
}

$type = (int)$row['share_type'];
if ($type === IShare::TYPE_ROOM) {
$roomToken = $row['share_with'];
Expand Down
Loading