Don't keep AST in memory

src/LanguageServer.php

@@ -153,7 +153,7 @@ class LanguageServer extends \AdvancedJsonRpc\Dispatcher
                 $shortName = substr($file, strlen($rootPath) + 1);
                 $this->client->window->logMessage(MessageType::INFO, "Parsing file $fileNum/$numTotalFiles: $shortName.");
 
-                $this->project->getDocument($uri)->parse(file_get_contents($file));
+                $this->project->getDocument($uri)->updateContent(file_get_contents($file), false);
 
                 Loop\setTimeout($processFile, 0);
             } else {

src/PhpDocument.php

@@ -9,6 +9,7 @@ use PhpParser\{Error, Comment, Node, ParserFactory, NodeTraverser, Lexer, Parser
 use PhpParser\PrettyPrinter\Standard as PrettyPrinter;
 use PhpParser\NodeVisitor\NameResolver;
 use Exception;
+use function LanguageServer\uriToPath;
 
 class PhpDocument
 {
@@ -85,83 +86,36 @@ class PhpDocument
     }
 
     /**
-     * Returns all symbols in this document.
+     * Returns true if the content of this document is being held in memory
      *
-     * @return SymbolInformation[]|null
+     * @return bool
      */
-    public function getSymbols()
+    public function isLoaded()
     {
-        if (!isset($this->definitions)) {
+        return isset($this->content);
-            return null;
-        }
-        $nodeSymbolKindMap = [
-            Node\Stmt\Class_::class           => SymbolKind::CLASS_,
-            Node\Stmt\Trait_::class           => SymbolKind::CLASS_,
-            Node\Stmt\Interface_::class       => SymbolKind::INTERFACE,
-            Node\Stmt\Namespace_::class       => SymbolKind::NAMESPACE,
-            Node\Stmt\Function_::class        => SymbolKind::FUNCTION,
-            Node\Stmt\ClassMethod::class      => SymbolKind::METHOD,
-            Node\Stmt\PropertyProperty::class => SymbolKind::PROPERTY,
-            Node\Const_::class                => SymbolKind::CONSTANT
-        ];
-        $symbols = [];
-        foreach ($this->definitions as $fqn => $node) {
-            $class = get_class($node);
-            if (!isset($nodeSymbolKindMap[$class])) {
-                continue;
-            }
-            $symbol = new SymbolInformation();
-            $symbol->kind = $nodeSymbolKindMap[$class];
-            $symbol->name = (string)$node->name;
-            $symbol->location = Location::fromNode($node);
-            $parts = preg_split('/(::|\\\\)/', $fqn);
-            array_pop($parts);
-            $symbol->containerName = implode('\\', $parts);
-            $symbols[] = $symbol;
-        }
-        return $symbols;
     }
 
     /**
-     * Returns symbols in this document filtered by query string.
+     * Loads the content from disk and saves statements and definitions in memory
-     *
-     * @param string $query The search query
-     * @return SymbolInformation[]|null
-     */
-    public function findSymbols(string $query)
-    {
-        $symbols = $this->getSymbols();
-        if ($symbols === null) {
-            return null;
-        }
-        if ($query === '') {
-            return $symbols;
-        }
-        return array_filter($symbols, function($symbol) use ($query) {
-            return stripos($symbol->name, $query) !== false;
-        });
-    }
-
-    /**
-     * Updates the content on this document.
-     *
-     * @param string $content
-     * @return void
-     */
-    public function updateContent(string $content)
-    {
-        $this->content = $content;
-        $this->parse($content);
-    }
-
-    /**
-     * Unloads the content from memory
      *
      * @return void
      */
-    public function removeContent()
+    public function load()
+    {
+        $this->updateContent(file_get_contents(uriToPath($this->getUri())), true);
+    }
+
+    /**
+     * Unloads the content, statements and definitions from memory
+     *
+     * @return void
+     */
+    public function unload()
     {
         unset($this->content);
+        unset($this->statements);
+        unset($this->definitions);
+        unset($this->references);
     }
 
     /**
@@ -169,10 +123,15 @@ class PhpDocument
      * that may have occured as diagnostics.
      *
      * @param string $content
+     * @param bool $keepInMemory Wether to keep content, statements and definitions in memory or only update project definitions
      * @return void
      */
-    public function parse(string $content)
+    public function updateContent(string $content, bool $keepInMemory = true)
     {
+        $keepInMemory = $keepInMemory || $this->isLoaded();
+        if ($keepInMemory) {
+            $this->content = $content;
+        }
         $stmts = null;
         $errors = [];
         try {
@@ -219,13 +178,15 @@ class PhpDocument
 
             $traverser->traverse($stmts);
 
-            $this->definitions = $definitionCollector->definitions;
             // Register this document on the project for all the symbols defined in it
             foreach ($definitionCollector->definitions as $fqn => $node) {
                 $this->project->addDefinitionDocument($fqn, $this);
             }
 
+            if ($keepInMemory) {
                 $this->statements = $stmts;
+                $this->definitions = $definitionCollector->definitions;
+            }
         }
     }
 
@@ -261,7 +222,7 @@ class PhpDocument
      *
      * @return Node[]
      */
-    public function getStatements()
+    public function &getStatements()
     {
         if (!isset($this->statements)) {
             $this->parse($this->getContent());
@@ -302,7 +263,21 @@ class PhpDocument
      */
     public function getDefinitionByFqn(string $fqn)
     {
-        return $this->definitions[$fqn] ?? null;
+        return $this->getDefinitions()[$fqn] ?? null;
+    }
+
+    /**
+     * Returns a map from fully qualified name (FQN) to Nodes defined in this document
+     *
+     * @return Node[]
+     * @throws Exception If the definitions are not loaded
+     */
+    public function &getDefinitions()
+    {
+        if (!isset($this->definitions)) {
+            throw new Exception('Definitions of this document are not loaded');
+        }
+        return $this->definitions;
     }
 
     /**
@@ -313,7 +288,7 @@ class PhpDocument
      */
     public function isDefined(string $fqn): bool
     {
-        return isset($this->definitions[$fqn]);
+        return isset($this->getDefinitions()[$fqn]);
     }
 
     /**
@@ -511,6 +486,9 @@ class PhpDocument
             return null;
         }
         $document = $this->project->getDefinitionDocument($fqn);
+        if (!$document->isLoaded()) {
+            $document->load();
+        }
         if (!isset($document)) {
             return null;
         }

src/Project.php

@@ -83,6 +83,17 @@ class Project
         return $this->definitions[$fqn] ?? null;
     }
 
+    /**
+     * Returns an associative array [string => PhpDocument]
+     * that maps fully qualified symbol names to loaded PhpDocuments
+     *
+     * @return PhpDocument[]
+     */
+    public function &getDefinitionDocuments()
+    {
+        return $this->definitions;
+    }
+
     /**
      * Returns true if the given FQN is defined in the project
      *
@@ -93,22 +104,4 @@ class Project
     {
         return isset($this->definitions[$fqn]);
     }
-
-    /**
-     * Finds symbols in all documents, filtered by query parameter.
-     *
-     * @param string $query
-     * @return SymbolInformation[]
-     */
-    public function findSymbols(string $query)
-    {
-        $queryResult = [];
-        foreach ($this->documents as $uri => $document) {
-            $documentQueryResult = $document->findSymbols($query);
-            if ($documentQueryResult !== null) {
-                $queryResult = array_merge($queryResult, $documentQueryResult);
-            }
-        }
-        return $queryResult;
-    }
 }

src/Protocol/SymbolInformation.php

@@ -3,6 +3,7 @@
 namespace LanguageServer\Protocol;
 
 use PhpParser\Node;
+use Exception;
 
 /**
  * Represents information about programming constructs like variables, classes,
@@ -37,4 +38,39 @@ class SymbolInformation
      * @var string|null
      */
     public $containerName;
+
+    /**
+     * Converts a Node to a SymbolInformation
+     *
+     * @param Node $node
+     * @param string $fqn If given, $containerName will be extracted from it
+     * @return self
+     */
+    public static function fromNode(Node $node, string $fqn = null)
+    {
+        $nodeSymbolKindMap = [
+            Node\Stmt\Class_::class           => SymbolKind::CLASS_,
+            Node\Stmt\Trait_::class           => SymbolKind::CLASS_,
+            Node\Stmt\Interface_::class       => SymbolKind::INTERFACE,
+            Node\Stmt\Namespace_::class       => SymbolKind::NAMESPACE,
+            Node\Stmt\Function_::class        => SymbolKind::FUNCTION,
+            Node\Stmt\ClassMethod::class      => SymbolKind::METHOD,
+            Node\Stmt\PropertyProperty::class => SymbolKind::PROPERTY,
+            Node\Const_::class                => SymbolKind::CONSTANT
+        ];
+        $class = get_class($node);
+        if (!isset($nodeSymbolKindMap[$class])) {
+            throw new Exception("Not a declaration node: $class");
+        }
+        $symbol = new self;
+        $symbol->kind = $nodeSymbolKindMap[$class];
+        $symbol->name = (string)$node->name;
+        $symbol->location = Location::fromNode($node);
+        if ($fqn !== null) {
+            $parts = preg_split('/(::|\\\\)/', $fqn);
+            array_pop($parts);
+            $symbol->containerName = implode('\\', $parts);
+        }
+        return $symbol;
+    }
 }

src/Server/TextDocument.php

@@ -14,7 +14,8 @@ use LanguageServer\Protocol\{
     Position,
     FormattingOptions,
     TextEdit,
-    Location
+    Location,
+    SymbolInformation
 };
 
 /**
@@ -49,7 +50,11 @@ class TextDocument
      */
     public function documentSymbol(TextDocumentIdentifier $textDocument): array
     {
-        return $this->project->getDocument($textDocument->uri)->getSymbols();
+        $symbols = [];
+        foreach ($this->project->getDocument($textDocument->uri)->getDefinitions() as $fqn => $node) {
+            $symbols[] = SymbolInformation::fromNode($node, $fqn);
+        }
+        return $symbols;
     }
 
     /**
@@ -87,7 +92,7 @@ class TextDocument
      */
     public function didClose(TextDocumentIdentifier $textDocument)
     {
-        $this->project->getDocument($textDocument->uri)->removeContent();
+        $this->project->getDocument($textDocument->uri)->unload();
     }
 
     /**

src/Server/Workspace.php

@@ -53,6 +53,15 @@ class Workspace
      */
     public function symbol(string $query): array
     {
-        return $this->project->findSymbols($query);
+        $symbols = [];
+        foreach ($this->project->getDefinitionDocuments() as $fqn => $document) {
+            if ($query === '' || stripos($fqn, $query) !== false) {
+                if (!$document->isLoaded()) {
+                    $document->load();
+                }
+                $symbols[] = SymbolInformation::fromNode($document->getDefinitionByFqn($fqn), $fqn);
+            }
+        }
+        return $symbols;
     }
 }

tests/PhpDocumentTest.php

@@ -28,9 +28,7 @@ class PhpDocumentTest extends TestCase
 
         $document->updateContent("<?php\n$\$a = 'foo';\n\$bar = 'baz';\n");
 
-        $symbols = $document->getSymbols();
+        $this->assertEquals([], $document->getDefinitions());
-
-        $this->assertEquals([], json_decode(json_encode($symbols), true));
     }
 
     public function testGetNodeAtPosition()

tests/ProjectTest.php

@@ -36,52 +36,4 @@ class ProjectTest extends TestCase
 
         $this->assertSame($document1, $document2);
     }
-
-    public function testFindSymbols()
-    {
-        $this->project->getDocument('file:///document1.php')->updateContent("<?php\nfunction foo() {}\nfunction bar() {}\n");
-        $this->project->getDocument('file:///document2.php')->updateContent("<?php\nfunction baz() {}\nfunction frob() {}\n");
-        $this->project->getDocument('invalid_file')->updateContent(file_get_contents(__DIR__ . '/../fixtures/invalid_file.php'));
-
-        $symbols = $this->project->findSymbols('ba');
-
-        $this->assertEquals([
-            [
-                'name' => 'bar',
-                'kind' => SymbolKind::FUNCTION,
-                'location' => [
-                    'uri' => 'file:///document1.php',
-                    'range' => [
-                        'start' => [
-                            'line' => 2,
-                            'character' => 0
-                        ],
-                        'end' => [
-                            'line' => 2,
-                            'character' => 17
-                        ]
-                    ]
-                ],
-                'containerName' => null
-            ],
-            [
-                'name' => 'baz',
-                'kind' => SymbolKind::FUNCTION,
-                'location' => [
-                    'uri' => 'file:///document2.php',
-                    'range' => [
-                        'start' => [
-                            'line' => 1,
-                            'character' => 0
-                        ],
-                        'end' => [
-                            'line' => 1,
-                            'character' => 17
-                        ]
-                    ]
-                ],
-                'containerName' => null
-            ]
-        ], json_decode(json_encode($symbols), true));
-    }
 }

tests/Server/Workspace/SymbolTest.php

@@ -1,13 +1,14 @@
 <?php
 declare(strict_types = 1);
 
-namespace LanguageServer\Tests\Server;
+namespace LanguageServer\Tests\Server\Workspace;
 
 use PHPUnit\Framework\TestCase;
 use LanguageServer\Tests\MockProtocolStream;
 use LanguageServer\{Server, Client, LanguageClient, Project, PhpDocument};
 use LanguageServer\Protocol\{TextDocumentItem, TextDocumentIdentifier, SymbolKind, DiagnosticSeverity, FormattingOptions};
 use AdvancedJsonRpc\{Request as RequestBody, Response as ResponseBody};
+use function LanguageServer\pathToUri;
 
 class SymbolTest extends TestCase
 {
@@ -16,13 +17,25 @@ class SymbolTest extends TestCase
      */
     private $workspace;
 
+    /**
+     * @var string
+     */
+    private $symbolsUri;
+
+    /**
+     * @var string
+     */
+    private $referencesUri;
+
     public function setUp()
     {
         $client = new LanguageClient(new MockProtocolStream());
         $project = new Project($client);
         $this->workspace = new Server\Workspace($project, $client);
-        $project->getDocument('symbols')->updateContent(file_get_contents(__DIR__ . '/../../../fixtures/symbols.php'));
+        $this->symbolsUri = pathToUri(realpath(__DIR__ . '/../../../fixtures/symbols.php'));
-        $project->getDocument('references')->updateContent(file_get_contents(__DIR__ . '/../../../fixtures/references.php'));
+        $this->referencesUri = pathToUri(realpath(__DIR__ . '/../../../fixtures/references.php'));
+        $project->getDocument($this->symbolsUri)->updateContent(file_get_contents($this->symbolsUri), false);
+        $project->getDocument($this->referencesUri)->updateContent(file_get_contents($this->referencesUri), false);
     }
 
     public function testEmptyQueryReturnsAllSymbols()
@@ -34,7 +47,7 @@ class SymbolTest extends TestCase
                 'name' => 'TEST_CONST',
                 'kind' => SymbolKind::CONSTANT,
                 'location' => [
-                    'uri' => 'symbols',
+                    'uri' => $this->symbolsUri,
                     'range' => [
                         'start' => [
                             'line' => 4,
@@ -52,7 +65,7 @@ class SymbolTest extends TestCase
                 'name' => 'TestClass',
                 'kind' => SymbolKind::CLASS_,
                 'location' => [
-                    'uri' => 'symbols',
+                    'uri' => $this->symbolsUri,
                     'range' => [
                         'start' => [
                             'line' => 6,
@@ -70,7 +83,7 @@ class SymbolTest extends TestCase
                 'name' => 'TEST_CLASS_CONST',
                 'kind' => SymbolKind::CONSTANT,
                 'location' => [
-                    'uri' => 'symbols',
+                    'uri' => $this->symbolsUri,
                     'range' => [
                         'start' => [
                             'line' => 8,
@@ -88,7 +101,7 @@ class SymbolTest extends TestCase
                 'name' => 'staticTestProperty',
                 'kind' => SymbolKind::PROPERTY,
                 'location' => [
-                    'uri' => 'symbols',
+                    'uri' => $this->symbolsUri,
                     'range' => [
                         'start' => [
                             'line' => 9,
@@ -106,7 +119,7 @@ class SymbolTest extends TestCase
                 'name' => 'testProperty',
                 'kind' => SymbolKind::PROPERTY,
                 'location' => [
-                    'uri' => 'symbols',
+                    'uri' => $this->symbolsUri,
                     'range' => [
                         'start' => [
                             'line' => 10,
@@ -124,7 +137,7 @@ class SymbolTest extends TestCase
                 'name' => 'staticTestMethod',
                 'kind' => SymbolKind::METHOD,
                 'location' => [
-                    'uri' => 'symbols',
+                    'uri' => $this->symbolsUri,
                     'range' => [
                         'start' => [
                             'line' => 12,
@@ -142,7 +155,7 @@ class SymbolTest extends TestCase
                 'name' => 'testMethod',
                 'kind' => SymbolKind::METHOD,
                 'location' => [
-                    'uri' => 'symbols',
+                    'uri' => $this->symbolsUri,
                     'range' => [
                         'start' => [
                             'line' => 17,
@@ -160,7 +173,7 @@ class SymbolTest extends TestCase
                 'name' => 'TestTrait',
                 'kind' => SymbolKind::CLASS_,
                 'location' => [
-                    'uri' => 'symbols',
+                    'uri' => $this->symbolsUri,
                     'range' => [
                         'start' => [
                             'line' => 23,
@@ -178,7 +191,7 @@ class SymbolTest extends TestCase
                 'name' => 'TestInterface',
                 'kind' => SymbolKind::INTERFACE,
                 'location' => [
-                    'uri' => 'symbols',
+                    'uri' => $this->symbolsUri,
                     'range' => [
                         'start' => [
                             'line' => 28,
@@ -196,7 +209,7 @@ class SymbolTest extends TestCase
                 'name' => 'test_function',
                 'kind' => SymbolKind::FUNCTION,
                 'location' => [
-                    'uri' => 'symbols',
+                    'uri' => $this->symbolsUri,
                     'range' => [
                         'start' => [
                             'line' => 33,
@@ -214,7 +227,7 @@ class SymbolTest extends TestCase
                 'name' => 'whatever',
                 'kind' => SymbolKind::FUNCTION,
                 'location' => [
-                    'uri' => 'references',
+                    'uri' => $this->referencesUri,
                     'range' => [
                         'start' => [
                             'line' => 15,
@@ -240,7 +253,7 @@ class SymbolTest extends TestCase
                 'name' => 'staticTestMethod',
                 'kind' => SymbolKind::METHOD,
                 'location' => [
-                    'uri' => 'symbols',
+                    'uri' => $this->symbolsUri,
                     'range' => [
                         'start' => [
                             'line' => 12,
@@ -258,7 +271,7 @@ class SymbolTest extends TestCase
                 'name' => 'testMethod',
                 'kind' => SymbolKind::METHOD,
                 'location' => [
-                    'uri' => 'symbols',
+                    'uri' => $this->symbolsUri,
                     'range' => [
                         'start' => [
                             'line' => 17,