From bedd1576365d14e57404a48b796bb2e8f0a4bef5 Mon Sep 17 00:00:00 2001 From: Felix Becker Date: Sat, 4 Feb 2017 00:20:38 +0100 Subject: [PATCH] Caching (#260) --- src/Cache/Cache.php | 29 ++++ src/Cache/ClientCache.php | 48 ++++++ src/Cache/FileSystemCache.php | 68 +++++++++ src/Client/XCache.php | 43 ++++++ src/Index/DependenciesIndex.php | 11 ++ src/Index/Index.php | 28 +++- src/Indexer.php | 222 ++++++++++++++++++++++++++++ src/LanguageClient.php | 8 + src/LanguageServer.php | 110 ++++---------- src/Protocol/ClientCapabilities.php | 7 + tests/LanguageServerTest.php | 6 +- 11 files changed, 496 insertions(+), 84 deletions(-) create mode 100644 src/Cache/Cache.php create mode 100644 src/Cache/ClientCache.php create mode 100644 src/Cache/FileSystemCache.php create mode 100644 src/Client/XCache.php create mode 100644 src/Indexer.php diff --git a/src/Cache/Cache.php b/src/Cache/Cache.php new file mode 100644 index 0000000..ebc5233 --- /dev/null +++ b/src/Cache/Cache.php @@ -0,0 +1,29 @@ + + */ + public function get(string $key): Promise; + + /** + * Sets a value in the cache + * + * @param string $key + * @param mixed $value + * @return Promise + */ + public function set(string $key, $value): Promise; +} diff --git a/src/Cache/ClientCache.php b/src/Cache/ClientCache.php new file mode 100644 index 0000000..bc91b97 --- /dev/null +++ b/src/Cache/ClientCache.php @@ -0,0 +1,48 @@ +client = $client; + } + + /** + * Gets a value from the cache + * + * @param string $key + * @return Promise + */ + public function get(string $key): Promise + { + return $this->client->xcache->get($key)->then('unserialize')->otherwise(function () { + // Ignore + }); + } + + /** + * Sets a value in the cache + * + * @param string $key + * @param mixed $value + * @return Promise + */ + public function set(string $key, $value): Promise + { + return $this->client->xcache->set($key, serialize($value))->otherwise(function () { + // Ignore + }); + } +} diff --git a/src/Cache/FileSystemCache.php b/src/Cache/FileSystemCache.php new file mode 100644 index 0000000..8a507f7 --- /dev/null +++ b/src/Cache/FileSystemCache.php @@ -0,0 +1,68 @@ +cacheDir = getenv('LOCALAPPDATA') . '\\PHP Language Server\\'; + } else if (getenv('XDG_CACHE_HOME')) { + $this->cacheDir = getenv('XDG_CACHE_HOME') . '/phpls/'; + } else { + $this->cacheDir = getenv('HOME') . '/.phpls/'; + } + } + + /** + * Gets a value from the cache + * + * @param string $key + * @return Promise + */ + public function get(string $key): Promise + { + try { + $file = $this->cacheDir . urlencode($key); + if (!file_exists($file)) { + return Promise\resolve(null); + } + return Promise\resolve(unserialize(file_get_contents($file))); + } catch (\Exception $e) { + return Promise\resolve(null); + } + } + + /** + * Sets a value in the cache + * + * @param string $key + * @param mixed $value + * @return Promise + */ + public function set(string $key, $value): Promise + { + try { + $file = $this->cacheDir . urlencode($key); + if (!file_exists($this->cacheDir)) { + mkdir($this->cacheDir); + } + file_put_contents($file, serialize($value)); + } finally { + return Promise\resolve(null); + } + } +} diff --git a/src/Client/XCache.php b/src/Client/XCache.php new file mode 100644 index 0000000..3004e58 --- /dev/null +++ b/src/Client/XCache.php @@ -0,0 +1,43 @@ +handler = $handler; + } + + /** + * @param string $key + * @return Promise + */ + public function get(string $key): Promise + { + return $this->handler->request('xcache/get', ['key' => $key]); + } + + /** + * @param string $key + * @param mixed $value + * @return Promise + */ + public function set(string $key, $value): Promise + { + return $this->handler->notify('xcache/set', ['key' => $key, 'value' => $value]); + } +} diff --git a/src/Index/DependenciesIndex.php b/src/Index/DependenciesIndex.php index aa5e06e..059cb7d 100644 --- a/src/Index/DependenciesIndex.php +++ b/src/Index/DependenciesIndex.php @@ -34,6 +34,17 @@ class DependenciesIndex extends AbstractAggregateIndex return $this->indexes[$packageName]; } + /** + * @param string $packageName + * @param Index $index + * @return void + */ + public function setDependencyIndex(string $packageName, Index $index) + { + $this->indexes[$packageName] = $index; + $this->registerIndex($index); + } + /** * @param string $packageName * @return void diff --git a/src/Index/Index.php b/src/Index/Index.php index 183e283..5c24813 100644 --- a/src/Index/Index.php +++ b/src/Index/Index.php @@ -10,7 +10,7 @@ use Sabre\Event\EmitterTrait; * Represents the index of a project or dependency * Serializable for caching */ -class Index implements ReadableIndex +class Index implements ReadableIndex, \Serializable { use EmitterTrait; @@ -185,4 +185,30 @@ class Index implements ReadableIndex } array_splice($this->references[$fqn], $index, 1); } + + /** + * @param string $serialized + * @return void + */ + public function unserialize($serialized) + { + $data = unserialize($serialized); + foreach ($data as $prop => $val) { + $this->$prop = $val; + } + } + + /** + * @param string $serialized + * @return string + */ + public function serialize() + { + return serialize([ + 'definitions' => $this->definitions, + 'references' => $this->references, + 'complete' => $this->complete, + 'staticComplete' => $this->staticComplete + ]); + } } diff --git a/src/Indexer.php b/src/Indexer.php new file mode 100644 index 0000000..df9655a --- /dev/null +++ b/src/Indexer.php @@ -0,0 +1,222 @@ +filesFinder = $filesFinder; + $this->rootPath = $rootPath; + $this->client = $client; + $this->cache = $cache; + $this->dependenciesIndex = $dependenciesIndex; + $this->sourceIndex = $sourceIndex; + $this->documentLoader = $documentLoader; + $this->composerLock = $composerLock; + } + + /** + * Will read and parse the passed source files in the project and add them to the appropiate indexes + * + * @return Promise + */ + public function index(): Promise + { + return coroutine(function () { + + $pattern = Path::makeAbsolute('**/*.php', $this->rootPath); + $uris = yield $this->filesFinder->find($pattern); + + $count = count($uris); + $startTime = microtime(true); + $this->client->window->logMessage(MessageType::INFO, "$count files total"); + + /** @var string[] */ + $source = []; + /** @var string[][] */ + $deps = []; + foreach ($uris as $uri) { + if ($this->composerLock !== null && preg_match('/\/vendor\/([^\/]+\/[^\/]+)\//', $uri, $matches)) { + // Dependency file + $packageName = $matches[1]; + if (!isset($deps[$packageName])) { + $deps[$packageName] = []; + } + $deps[$packageName][] = $uri; + } else { + // Source file + $source[] = $uri; + } + } + + // Index source + // Definitions and static references + $this->client->window->logMessage(MessageType::INFO, 'Indexing project for definitions and static references'); + yield $this->indexFiles($source); + $this->sourceIndex->setStaticComplete(); + // Dynamic references + $this->client->window->logMessage(MessageType::INFO, 'Indexing project for dynamic references'); + yield $this->indexFiles($source); + $this->sourceIndex->setComplete(); + + // Index dependencies + $this->client->window->logMessage(MessageType::INFO, count($deps) . ' Packages'); + foreach ($deps as $packageName => $files) { + // Find version of package and check cache + $packageKey = null; + $cacheKey = null; + $index = null; + foreach (array_merge($this->composerLock->packages, $this->composerLock->{'packages-dev'}) as $package) { + // Check if package name matches and version is absolute + // Dynamic constraints are not cached, because they can change every time + $packageVersion = ltrim($package->version, 'v'); + if ($package->name === $packageName && strpos($packageVersion, 'dev') === false) { + $packageKey = $packageName . ':' . $packageVersion; + $cacheKey = self::CACHE_VERSION . ':' . $packageKey; + // Check cache + $index = yield $this->cache->get($cacheKey); + break; + } + } + if ($index !== null) { + // Cache hit + $this->dependenciesIndex->setDependencyIndex($packageName, $index); + $this->client->window->logMessage(MessageType::INFO, "Restored $packageKey from cache"); + } else { + // Cache miss + $index = $this->dependenciesIndex->getDependencyIndex($packageName); + + // Index definitions and static references + $this->client->window->logMessage(MessageType::INFO, 'Indexing ' . ($packageKey ?? $packageName) . ' for definitions and static references'); + yield $this->indexFiles($files); + $index->setStaticComplete(); + + // Index dynamic references + $this->client->window->logMessage(MessageType::INFO, 'Indexing ' . ($packageKey ?? $packageName) . ' for dynamic references'); + yield $this->indexFiles($files); + $index->setComplete(); + + // If we know the version (cache key), save index for the dependency in the cache + if ($cacheKey !== null) { + $this->client->window->logMessage(MessageType::INFO, "Storing $packageKey in cache"); + $this->cache->set($cacheKey, $index); + } + } + } + + $duration = (int)(microtime(true) - $startTime); + $mem = (int)(memory_get_usage(true) / (1024 * 1024)); + $this->client->window->logMessage( + MessageType::INFO, + "All $count PHP files parsed in $duration seconds. $mem MiB allocated." + ); + }); + } + + /** + * @param array $files + * @return Promise + */ + private function indexFiles(array $files): Promise + { + return coroutine(function () use ($files) { + foreach ($files as $i => $uri) { + // Skip open documents + if ($this->documentLoader->isOpen($uri)) { + continue; + } + + // Give LS to the chance to handle requests while indexing + yield timeout(); + $this->client->window->logMessage(MessageType::LOG, "Parsing $uri"); + try { + $document = yield $this->documentLoader->load($uri); + if (!$document->isVendored()) { + $this->client->textDocument->publishDiagnostics($uri, $document->getDiagnostics()); + } + } catch (ContentTooLargeException $e) { + $this->client->window->logMessage( + MessageType::INFO, + "Ignoring file {$uri} because it exceeds size limit of {$e->limit} bytes ({$e->size})" + ); + } catch (\Exception $e) { + $this->client->window->logMessage(MessageType::ERROR, "Error parsing $uri: " . (string)$e); + } + } + }); + } +} diff --git a/src/LanguageClient.php b/src/LanguageClient.php index d21a9aa..c68170f 100644 --- a/src/LanguageClient.php +++ b/src/LanguageClient.php @@ -28,6 +28,13 @@ class LanguageClient */ public $workspace; + /** + * Handles xcache/* methods + * + * @var Client\XCache + */ + public $xcache; + public function __construct(ProtocolReader $reader, ProtocolWriter $writer) { $handler = new ClientHandler($reader, $writer); @@ -36,5 +43,6 @@ class LanguageClient $this->textDocument = new Client\TextDocument($handler, $mapper); $this->window = new Client\Window($handler); $this->workspace = new Client\Workspace($handler, $mapper); + $this->xcache = new Client\XCache($handler); } } diff --git a/src/LanguageServer.php b/src/LanguageServer.php index 7e856b1..035e663 100644 --- a/src/LanguageServer.php +++ b/src/LanguageServer.php @@ -17,13 +17,13 @@ use LanguageServer\Protocol\{ use LanguageServer\FilesFinder\{FilesFinder, ClientFilesFinder, FileSystemFilesFinder}; use LanguageServer\ContentRetriever\{ContentRetriever, ClientContentRetriever, FileSystemContentRetriever}; use LanguageServer\Index\{DependenciesIndex, GlobalIndex, Index, ProjectIndex, StubsIndex}; +use LanguageServer\Cache\{FileSystemCache, ClientCache}; use AdvancedJsonRpc; use Sabre\Event\{Loop, Promise}; use function Sabre\Event\coroutine; use Exception; use Throwable; use Webmozart\PathUtil\Path; -use Webmozart\Glob\Glob; use Sabre\Uri; class LanguageServer extends AdvancedJsonRpc\Dispatcher @@ -202,23 +202,39 @@ class LanguageServer extends AdvancedJsonRpc\Dispatcher if ($rootPath !== null) { yield $this->beforeIndex($rootPath); - $this->index($rootPath)->otherwise('\\LanguageServer\\crash'); + + // Find composer.json + if ($this->composerJson === null) { + $composerJsonFiles = yield $this->filesFinder->find(Path::makeAbsolute('**/composer.json', $rootPath)); + if (!empty($composerJsonFiles)) { + $this->composerJson = json_decode(yield $this->contentRetriever->retrieve($composerJsonFiles[0])); + } + } + + // Find composer.lock + if ($this->composerLock === null) { + $composerLockFiles = yield $this->filesFinder->find(Path::makeAbsolute('**/composer.lock', $rootPath)); + if (!empty($composerLockFiles)) { + $this->composerLock = json_decode(yield $this->contentRetriever->retrieve($composerLockFiles[0])); + } + } + + $cache = $capabilities->xcacheProvider ? new ClientCache($this->client) : new FileSystemCache; + + // Index in background + $indexer = new Indexer( + $this->filesFinder, + $rootPath, + $this->client, + $cache, + $dependenciesIndex, + $sourceIndex, + $this->documentLoader, + $this->composerLock + ); + $indexer->index()->otherwise('\\LanguageServer\\crash'); } - // Find composer.json - if ($this->composerJson === null) { - $composerJsonFiles = yield $this->filesFinder->find(Path::makeAbsolute('**/composer.json', $rootPath)); - if (!empty($composerJsonFiles)) { - $this->composerJson = json_decode(yield $this->contentRetriever->retrieve($composerJsonFiles[0])); - } - } - // Find composer.lock - if ($this->composerLock === null) { - $composerLockFiles = yield $this->filesFinder->find(Path::makeAbsolute('**/composer.lock', $rootPath)); - if (!empty($composerLockFiles)) { - $this->composerLock = json_decode(yield $this->contentRetriever->retrieve($composerLockFiles[0])); - } - } if ($this->textDocument === null) { $this->textDocument = new Server\TextDocument( @@ -298,66 +314,4 @@ class LanguageServer extends AdvancedJsonRpc\Dispatcher protected function beforeIndex(string $rootPath) { } - - /** - * Will read and parse the passed source files in the project and add them to the appropiate indexes - * - * @param string $rootPath - * @return Promise - */ - protected function index(string $rootPath): Promise - { - return coroutine(function () use ($rootPath) { - - $pattern = Path::makeAbsolute('**/*.php', $rootPath); - $uris = yield $this->filesFinder->find($pattern); - - $count = count($uris); - - $startTime = microtime(true); - - foreach (['Collecting definitions and static references', 'Collecting dynamic references'] as $run => $text) { - $this->client->window->logMessage(MessageType::INFO, $text); - foreach ($uris as $i => $uri) { - if ($this->documentLoader->isOpen($uri)) { - continue; - } - - // Give LS to the chance to handle requests while indexing - yield timeout(); - $this->client->window->logMessage( - MessageType::LOG, - "Parsing file $i/$count: {$uri}" - ); - try { - $document = yield $this->documentLoader->load($uri); - if (!$document->isVendored()) { - $this->client->textDocument->publishDiagnostics($uri, $document->getDiagnostics()); - } - } catch (ContentTooLargeException $e) { - $this->client->window->logMessage( - MessageType::INFO, - "Ignoring file {$uri} because it exceeds size limit of {$e->limit} bytes ({$e->size})" - ); - } catch (Exception $e) { - $this->client->window->logMessage( - MessageType::ERROR, - "Error parsing file {$uri}: " . (string)$e - ); - } - } - if ($run === 0) { - $this->projectIndex->setStaticComplete(); - } else { - $this->projectIndex->setComplete(); - } - $duration = (int)(microtime(true) - $startTime); - $mem = (int)(memory_get_usage(true) / (1024 * 1024)); - $this->client->window->logMessage( - MessageType::INFO, - "All $count PHP files parsed in $duration seconds. $mem MiB allocated." - ); - } - }); - } } diff --git a/src/Protocol/ClientCapabilities.php b/src/Protocol/ClientCapabilities.php index 42137e9..5228c7d 100644 --- a/src/Protocol/ClientCapabilities.php +++ b/src/Protocol/ClientCapabilities.php @@ -17,4 +17,11 @@ class ClientCapabilities * @var bool|null */ public $xcontentProvider; + + /** + * The client supports xcache/* requests + * + * @var bool|null + */ + public $xcacheProvider; } diff --git a/tests/LanguageServerTest.php b/tests/LanguageServerTest.php index 6bfa3c9..38b1fa1 100644 --- a/tests/LanguageServerTest.php +++ b/tests/LanguageServerTest.php @@ -104,11 +104,7 @@ class LanguageServerTest extends TestCase $promise->reject(new Exception($msg->body->params->message)); } } else if (strpos($msg->body->params->message, 'All 25 PHP files parsed') !== false) { - if ($run === 1) { - $run++; - } else { - $promise->fulfill(); - } + $promise->fulfill(); } } });