Coverage for C:\Repos\leo-editor\leo\plugins\importers\linescanner.py: 65%

717 statements  

« prev     ^ index     » next       coverage.py v6.4, created at 2022-05-24 10:21 -0500

1#@+leo-ver=5-thin 

2#@+node:ekr.20161108125620.1: * @file ../plugins/importers/linescanner.py 

3#@+<< linescanner docstring >> 

4#@+node:ekr.20161108125805.1: ** << linescanner docstring >> 

5""" 

6#@@language rest 

7#@@wrap 

8 

9**Overview** 

10 

11Leo's import infrastructure in `leoImport.py` instantiates the 

12Importer instance and calls `i.run`, which calls `i.scan_lines`. 

13 

14New importers copy entire lines from the input file to Leo nodes. This 

15makes the new importers much less error prone than the legacy 

16(character-by-character) importers. 

17 

18New importers know *nothing* about parsing. They know only about how to 

19scan tokens *accurately*. 

20 

21**Writing a new importer** 

22 

23Just run the importer;; abbreviation! 

24 

25To make the importer importer;; functional you must: 

26 

271. Copy it from leoSettings (@settings-->Abbreviations-->@outline-data tree-abbreviations) 

28 to the corresponding location in myLeoSettings.leo. 

29 

302. Make sure @bool scripting-abbreviations is True in myLeoSettings.leo. 

31 

32**Using the abbreviation** 

33 

341. Just type importer;; in the body pane of an empty node. 

35 

36A dialog will prompt you for the name of the language. Suppose you type x. 

37 

382. Now you will be prompted for to fill in the first field:: 

39 

40 'extensions': [comma-separated lists of extensions, with leading periods], 

41 

42The italicized field will be highlighted. Type '.x' (including quotes) followed by two commas. 

43 

443. You will then be prompted to fill in the second field:: 

45 

46 strict = True leading whitespace is significant. Otherwise False, 

47 

48Again, the italicized field will be highlighted. 

49 

50Type False, followed by two commas. 

51 

524. You will then be prompted for the last field:: 

53 

54 ### Examples: 

55 # self.indent # for python, coffeescript. 

56 # self.curlies 

57 # (self, curlies, self.parens) 

58 return level 

59 

60Only "level" is highlighted. The comments provide some hints about what to type. 

61 

62Let's type "self.curlies" followed by two commas. 

63 

645. Nothing more is highlighted, so that's it! No more substitutions remain. 

65 The outline is ready to use! 

66 

67Take a look at the result. The new tree is an almost complete @@file node 

68for the importer. Subtrees contain an X_Importer class and an X_ScanState 

69class. Docstrings, ctors and __repr__ methods are complete. 

70 

71Note: The generated tree contain ### comments showing where more work may 

72be needed. I might remove the need for some of them, but there is no great 

73need to do so. 

74 

75""" 

76#@-<< linescanner docstring >> 

77#@+<< linescanner imports >> 

78#@+node:ekr.20161108130715.1: ** << linescanner imports >> 

79import io 

80import re 

81from typing import Any, Dict, List 

82from leo.core import leoGlobals as g 

83StringIO = io.StringIO 

84#@-<< linescanner imports >> 

85#@+others 

86#@+node:ekr.20161108155730.1: ** class Importer 

87class Importer: 

88 """ 

89 The new, unified, simplified, interface to Leo's importer code. 

90 

91 Eventually, all importers will create use this class. 

92 """ 

93 

94 #@+others 

95 #@+node:ekr.20161108155925.1: *3* i.__init__ & reloadSettings 

96 def __init__(self, 

97 importCommands, 

98 gen_refs=False, # True: generate section references, 

99 language=None, # For @language directive. 

100 name=None, # The kind of importer, usually the same as language 

101 state_class=None, # For i.scan_line 

102 strict=False, 

103 **kwargs 

104 ): 

105 """ 

106 Importer.__init__: New in Leo 6.1.1: ic and c may be None for unit tests. 

107 """ 

108 # Copies of args... 

109 self.importCommands = ic = importCommands 

110 self.c = c = ic and ic.c 

111 self.encoding = ic and ic.encoding or 'utf-8' 

112 self.gen_refs = gen_refs 

113 self.language = language or name # For the @language directive. 

114 self.name = name or language 

115 language = self.language 

116 name = self.name 

117 assert language and name 

118 assert self.language and self.name 

119 self.state_class = state_class 

120 self.strict = strict # True: leading whitespace is significant. 

121 

122 # Set from ivars... 

123 self.has_decls = name not in ('xml', 'org-mode', 'vimoutliner') 

124 self.is_rst = name in ('rst',) 

125 self.tree_type = ic.treeType if c else None # '@root', '@file', etc. 

126 

127 # Constants... 

128 if ic: 

129 data = g.set_delims_from_language(self.name) 

130 self.single_comment, self.block1, self.block2 = data 

131 else: 

132 self.single_comment, self.block1, self.block2 = '//', '/*', '*/' # Javascript. 

133 if ic: 

134 self.escape = c.atFileCommands.underindentEscapeString 

135 self.escape_string = r'%s([0-9]+)\.' % re.escape(self.escape) 

136 # m.group(1) is the unindent value. 

137 self.escape_pattern = re.compile(self.escape_string) 

138 self.ScanState = ScanState # Must be set by subclasses that use general_scan_line. 

139 self.tab_width = 0 # Must be set in run, using self.root. 

140 self.ws_pattern = re.compile(r'^\s*$|^\s*%s' % (self.single_comment or '')) 

141 

142 # Settings... 

143 self.reloadSettings() 

144 

145 # State vars. 

146 self.errors = 0 

147 if ic: 

148 ic.errors = 0 # Required. 

149 self.parse_body = False 

150 # Keys are headlines. Values are disambiguating number. 

151 self.refs_dict: Dict[str, int] = {} 

152 self.root = None 

153 self.skip = 0 # A skip count for x.gen_lines & its helpers. 

154 self.vnode_info: Dict[str, Any] = {} 

155 self.ws_error = False 

156 

157 def reloadSettings(self): 

158 c = self.c 

159 if not c: 

160 return 

161 getBool = c.config.getBool 

162 c.registerReloadSettings(self) 

163 # self.at_auto_separate_non_def_nodes = False 

164 self.add_context = getBool("add-context-to-headlines") 

165 self.add_file_context = getBool("add-file-context-to-headlines") 

166 self.at_auto_warns_about_leading_whitespace = getBool('at_auto_warns_about_leading_whitespace') 

167 self.warn_about_underindented_lines = True 

168 

169 #@+node:ekr.20161110042512.1: *3* i.Convenience methods for vnode_info dict 

170 def add_line(self, p, s, tag=None): 

171 """Append the line s to p.v._import_lines.""" 

172 assert s and isinstance(s, str), (repr(s), g.callers()) 

173 self.vnode_info[p.v]['lines'].append(s) 

174 

175 def extend_lines(self, p, lines): 

176 self.vnode_info[p.v]['lines'].extend(list(lines)) 

177 

178 def get_lines(self, p): 

179 return self.vnode_info[p.v]['lines'] 

180 

181 def has_lines(self, p): 

182 d = self.vnode_info.get(p.v) 

183 return d is not None and d.get('lines') is not None 

184 

185 def prepend_lines(self, p, lines): 

186 self.vnode_info[p.v]['lines'] = list(lines) + self.vnode_info[p.v]['lines'] 

187 

188 def set_lines(self, p, lines): 

189 self.vnode_info[p.v]['lines'] = list(lines) 

190 #@+node:ekr.20161108131153.7: *3* i.Overrides 

191 # These can be overridden in subclasses. 

192 #@+node:ekr.20161108131153.8: *4* i.adjust_parent 

193 def adjust_parent(self, parent, headline): 

194 """Return the effective parent. 

195 

196 This is overridden by the RstScanner class.""" 

197 return parent 

198 #@+node:ekr.20161108131153.9: *4* i.clean_headline 

199 def clean_headline(self, s, p=None): 

200 """ 

201 Return the cleaned version headline s. 

202 Will typically be overridden in subclasses. 

203 """ 

204 return s.strip() 

205 #@+node:ekr.20161110173058.1: *4* i.clean_nodes 

206 def clean_nodes(self, parent): 

207 """ 

208 Clean all nodes in parent's tree. 

209 Subclasses override this as desired. 

210 See perl_i.clean_nodes for an examplle. 

211 """ 

212 pass 

213 #@+node:ekr.20161120022121.1: *3* i.Scanning & scan tables 

214 #@+node:ekr.20161128025508.1: *4* i.get_new_dict 

215 #@@nobeautify 

216 

217 def get_new_dict(self, context): 

218 """ 

219 Return a *general* state dictionary for the given context. 

220 Subclasses may override... 

221 """ 

222 comment, block1, block2 = self.single_comment, self.block1, self.block2 

223 

224 def add_key(d, pattern, data): 

225 key = pattern[0] 

226 aList = d.get(key,[]) 

227 aList.append(data) 

228 d[key] = aList 

229 

230 d: Dict[str, List[Any]] 

231 

232 if context: 

233 d = { 

234 # key kind pattern ends? 

235 '\\': [('len+1', '\\', None),], 

236 '"': [('len', '"', context == '"'),], 

237 "'": [('len', "'", context == "'"),], 

238 } 

239 if block1 and block2: 

240 add_key(d, block2, ('len', block2, True)) 

241 else: 

242 # Not in any context. 

243 d = { 

244 # key kind pattern new-ctx deltas 

245 '\\':[('len+1', '\\', context, None),], 

246 '"': [('len', '"', '"', None),], 

247 "'": [('len', "'", "'", None),], 

248 '{': [('len', '{', context, (1,0,0)),], 

249 '}': [('len', '}', context, (-1,0,0)),], 

250 '(': [('len', '(', context, (0,1,0)),], 

251 ')': [('len', ')', context, (0,-1,0)),], 

252 '[': [('len', '[', context, (0,0,1)),], 

253 ']': [('len', ']', context, (0,0,-1)),], 

254 } 

255 if comment: 

256 add_key(d, comment, ('all', comment, '', None)) 

257 if block1 and block2: 

258 add_key(d, block1, ('len', block1, block1, None)) 

259 return d 

260 #@+node:ekr.20161113135037.1: *4* i.get_table 

261 #@@nobeautify 

262 cached_scan_tables: Dict[str, Any] = {} 

263 

264 def get_table(self, context): 

265 """ 

266 Return the state table for the given context. 

267 

268 This method handles caching. x.get_new_table returns the actual table. 

269 """ 

270 # Bug fix: must keep tables separate. 

271 key = '%s.%s' % (self.name, context) 

272 table = self.cached_scan_tables.get(key) 

273 if table: 

274 return table 

275 table = self.get_new_dict(context) 

276 self.cached_scan_tables[key] = table 

277 return table 

278 #@+node:ekr.20161128025444.1: *4* i.scan_dict 

279 def scan_dict(self, context, i, s, d): 

280 """ 

281 i.scan_dict: Scan at position i of s with the give context and dict. 

282 Return the 6-tuple: (new_context, i, delta_c, delta_p, delta_s, bs_nl) 

283 """ 

284 found = False 

285 delta_c = delta_p = delta_s = 0 

286 ch = s[i] 

287 aList = d.get(ch) 

288 if aList and context: 

289 # In context. 

290 for data in aList: 

291 kind, pattern, ends = data 

292 if self.match(s, i, pattern): 

293 if ends is None: 

294 found = True 

295 new_context = context 

296 break 

297 elif ends: 

298 found = True 

299 new_context = '' 

300 break 

301 else: 

302 pass # Ignore this match. 

303 elif aList: 

304 # Not in context. 

305 for data in aList: 

306 kind, pattern, new_context, deltas = data 

307 if self.match(s, i, pattern): 

308 found = True 

309 if deltas: 

310 delta_c, delta_p, delta_s = deltas 

311 break 

312 if found: 

313 if kind == 'all': 

314 i = len(s) 

315 elif kind == 'len+1': 

316 i += (len(pattern) + 1) 

317 else: 

318 assert kind == 'len', (kind, self.name) 

319 i += len(pattern) 

320 bs_nl = pattern == '\\\n' 

321 return new_context, i, delta_c, delta_p, delta_s, bs_nl 

322 # 

323 # No match: stay in present state. All deltas are zero. 

324 new_context = context 

325 return new_context, i + 1, 0, 0, 0, False 

326 #@+node:ekr.20161108170435.1: *4* i.scan_line 

327 def scan_line(self, s, prev_state): 

328 """ 

329 A generalized scan-line method. 

330 

331 SCAN STATE PROTOCOL: 

332 

333 The Importer class should have a state_class ivar that references a 

334 **state class**. This class probably should *not* be subclass of the 

335 ScanState class, but it should observe the following protocol: 

336 

337 1. The state class's ctor must have the following signature: 

338 

339 def __init__(self, d) 

340 

341 2. The state class must have an update method. 

342 """ 

343 # This dict allows new data to be added without changing ScanState signatures. 

344 d = { 

345 'indent': self.get_int_lws(s), 

346 'is_ws_line': self.is_ws_line(s), 

347 'prev': prev_state, 

348 's': s, 

349 } 

350 new_state = self.state_class(d) 

351 i = 0 

352 while i < len(s): 

353 progress = i 

354 context = new_state.context 

355 table = self.get_table(context) 

356 data = self.scan_dict(context, i, s, table) 

357 i = new_state.update(data) 

358 assert progress < i 

359 return new_state 

360 #@+node:ekr.20161114024119.1: *4* i.test_scan_state 

361 def test_scan_state(self, tests, State): 

362 """ 

363 Test x.scan_line or i.scan_line. 

364 

365 `tests` is a list of g.Bunches with 'line' and 'ctx' fields. 

366 

367 A typical @command test: 

368 

369 if c.isChanged(): c.save() 

370 < < imp.reload importers.linescanner and importers.python > > 

371 importer = py.Py_Importer(c.importCommands) 

372 importer.test_scan_state(tests, Python_ScanState) 

373 """ 

374 assert self.single_comment == '#', self.single_comment 

375 table = self.get_table(context='') 

376 contexts = self.all_contexts(table) 

377 for bunch in tests: 

378 assert bunch.line is not None 

379 line = bunch.line 

380 ctx = getattr(bunch, 'ctx', None) 

381 if ctx: # Test one transition. 

382 ctx_in, ctx_out = ctx 

383 prev_state = State() 

384 prev_state.context = ctx_in 

385 new_state = self.scan_line(line, prev_state) 

386 new_context = new_state.context 

387 assert new_context == ctx_out, ( 

388 'FAIL1:\nline: %r\ncontext: %r new_context: %r ctx_out: %r\n%s\n%s' % ( 

389 line, ctx_in, new_context, ctx_out, prev_state, new_state)) 

390 else: # Test all transitions. 

391 for context in contexts: 

392 prev_state = State() 

393 prev_state.context = context 

394 new_state = self.scan_line(line, prev_state) 

395 assert new_state.context == context, ( 

396 'FAIL2:\nline: %r\ncontext: %r new_context: %r\n%s\n%s' % ( 

397 line, context, new_state.context, prev_state, new_state)) 

398 #@+node:ekr.20161108165530.1: *3* i.The pipeline 

399 #@+node:ekr.20161108131153.10: *4* i.run (driver) & helers 

400 def run(self, s, parent, parse_body=False): 

401 """The common top-level code for all scanners.""" 

402 c = self.c 

403 # Fix #449: Cloned @auto nodes duplicates section references. 

404 if parent.isCloned() and parent.hasChildren(): 

405 return None 

406 self.root = root = parent.copy() 

407 self.file_s = s 

408 # Init the error/status info. 

409 self.errors = 0 

410 self.parse_body = parse_body 

411 # Check for intermixed blanks and tabs. 

412 self.tab_width = c.getTabWidth(p=root) 

413 lines = g.splitLines(s) 

414 ws_ok = self.check_blanks_and_tabs(lines) # Only issues warnings. 

415 # Regularize leading whitespace 

416 if not ws_ok: 

417 lines = self.regularize_whitespace(lines) 

418 # Generate the nodes, including directives and section references. 

419 # Completely generate all nodes. 

420 self.generate_nodes(lines, parent) 

421 # Check the generated nodes. 

422 # Return True if the result is equivalent to the original file. 

423 if parse_body: 

424 ok = self.errors == 0 # Work around problems with directives. 

425 else: 

426 ok = self.errors == 0 and self.check(s, parent) 

427 # Insert an @ignore directive if there were any serious problems. 

428 if not ok: 

429 self.insert_ignore_directive(parent) 

430 # Importers should never dirty the outline. 

431 for p in root.self_and_subtree(): 

432 p.clearDirty() 

433 # #1451: Do not change the outline's change status. 

434 return ok # For unit tests. 

435 #@+node:ekr.20161108131153.14: *5* i.regularize_whitespace 

436 def regularize_whitespace(self, lines): 

437 """ 

438 Regularize leading whitespace in s: 

439 Convert tabs to blanks or vice versa depending on the @tabwidth in effect. 

440 """ 

441 kind = 'tabs' if self.tab_width > 0 else 'blanks' 

442 kind2 = 'blanks' if self.tab_width > 0 else 'tabs' 

443 fn = g.shortFileName(self.root.h) 

444 count, result, tab_width = 0, [], self.tab_width 

445 self.ws_error = False # 2016/11/23 

446 if tab_width < 0: # Convert tabs to blanks. 

447 for n, line in enumerate(lines): 

448 i, w = g.skip_leading_ws_with_indent(line, 0, tab_width) 

449 # Use negative width. 

450 s = g.computeLeadingWhitespace(w, -abs(tab_width)) + line[i:] 

451 if s != line: 

452 count += 1 

453 result.append(s) 

454 elif tab_width > 0: # Convert blanks to tabs. 

455 for n, line in enumerate(lines): 

456 # Use positive width. 

457 s = g.optimizeLeadingWhitespace(line, abs(tab_width)) 

458 if s != line: 

459 count += 1 

460 result.append(s) 

461 if count: 

462 self.ws_error = True # A flag to check. 

463 if not g.unitTesting: 

464 # g.es_print('Warning: Intermixed tabs and blanks in', fn) 

465 # g.es_print('Perfect import test will ignoring leading whitespace.') 

466 g.es('changed leading %s to %s in %s line%s in %s' % ( 

467 kind2, kind, count, g.plural(count), fn)) 

468 if g.unitTesting: # Sets flag for unit tests. 

469 self.report('changed %s lines' % count) 

470 return result 

471 #@+node:ekr.20161111024447.1: *5* i.generate_nodes 

472 def generate_nodes(self, lines, parent): 

473 """ 

474 A three-stage pipeline to generate all imported nodes. 

475 """ 

476 # Stage 1: generate nodes. 

477 # After this stage, the p.v._import_lines list contains p's future body text. 

478 if isinstance(lines, str): 

479 raise ValueError 

480 self.gen_lines(lines, parent) 

481 # 

482 # Optional Stage 2, consisting of zero or more sub-stages. 

483 # Subclasses may freely override this method, **provided** 

484 # that all substages use the API for setting body text. 

485 # Changing p.b directly will cause asserts to fail in i.finish(). 

486 self.post_pass(parent) 

487 # 

488 # Stage 3: Put directives in the root node and set p.b for all nodes. 

489 # 

490 # Subclasses should never need to override this stage. 

491 self.finish(parent) 

492 #@+node:ekr.20161108131153.11: *4* State 0: i.check_blanks_and_tabs 

493 def check_blanks_and_tabs(self, lines): 

494 """Check for intermixed blank & tabs.""" 

495 # Do a quick check for mixed leading tabs/blanks. 

496 fn = g.shortFileName(self.root.h) 

497 w = self.tab_width 

498 blanks = tabs = 0 

499 for s in lines: 

500 lws = self.get_str_lws(s) 

501 blanks += lws.count(' ') 

502 tabs += lws.count('\t') 

503 # Make sure whitespace matches @tabwidth directive. 

504 if w < 0: 

505 ok = tabs == 0 

506 message = 'tabs found with @tabwidth %s in %s' % (w, fn) 

507 elif w > 0: 

508 ok = blanks == 0 

509 message = 'blanks found with @tabwidth %s in %s' % (w, fn) 

510 if ok: 

511 ok = (blanks == 0 or tabs == 0) 

512 message = 'intermixed blanks and tabs in: %s' % (fn) 

513 if not ok: 

514 if g.unitTesting: 

515 self.report(message) 

516 else: 

517 g.es(message) 

518 return ok 

519 #@+node:ekr.20161108160409.1: *4* Stage 1: i.gen_lines & helpers 

520 def gen_lines(self, lines, parent): 

521 """ 

522 Non-recursively parse all lines of s into parent, creating descendant 

523 nodes as needed. 

524 """ 

525 trace = 'importers' in g.app.debug 

526 tail_p = None 

527 prev_state = self.state_class() 

528 target = Target(parent, prev_state) 

529 stack = [target, target] 

530 self.vnode_info = { 

531 # Keys are vnodes, values are inner dicts. 

532 parent.v: { 

533 'lines': [], 

534 } 

535 } 

536 if g.unitTesting: 

537 g.vnode_info = self.vnode_info # A hack. 

538 

539 self.skip = 0 

540 for i, line in enumerate(lines): 

541 new_state = self.scan_line(line, prev_state) 

542 top = stack[-1] 

543 # g.trace(new_state.level(), f"{new_state.level() < top.state.level():1}", repr(line)) 

544 if trace: 

545 g.trace('%d %d %s' % ( 

546 self.starts_block(i, lines, new_state, prev_state), 

547 self.ends_block(line, new_state, prev_state, stack), 

548 line.rstrip())) 

549 if self.skip > 0: 

550 self.skip -= 1 

551 elif self.is_ws_line(line): 

552 p = tail_p or top.p 

553 self.add_line(p, line) 

554 elif self.starts_block(i, lines, new_state, prev_state): 

555 tail_p = None 

556 self.start_new_block(i, lines, new_state, prev_state, stack) 

557 elif self.ends_block(line, new_state, prev_state, stack): 

558 tail_p = self.end_block(line, new_state, stack) 

559 else: 

560 p = tail_p or top.p 

561 self.add_line(p, line) 

562 prev_state = new_state 

563 #@+node:ekr.20161108160409.7: *5* i.create_child_node 

564 def create_child_node(self, parent, line, headline): 

565 """Create a child node of parent.""" 

566 child = parent.insertAsLastChild() 

567 self.vnode_info[child.v] = { 

568 'lines': [], 

569 } 

570 if line: 

571 self.add_line(child, line) 

572 assert isinstance(headline, str), repr(headline) 

573 child.h = headline.strip() 

574 return child 

575 #@+node:ekr.20161119130337.1: *5* i.cut_stack 

576 def cut_stack(self, new_state, stack): 

577 """Cut back the stack until stack[-1] matches new_state.""" 

578 

579 def underflow(n): 

580 g.trace(n) 

581 g.trace(new_state) 

582 g.printList(stack) 

583 

584 # assert len(stack) > 1 # Fail on entry. 

585 if len(stack) <= 1: 

586 return underflow(0) 

587 while stack: 

588 top_state = stack[-1].state 

589 if new_state.level() < top_state.level(): 

590 if len(stack) > 1: 

591 stack.pop() 

592 else: 

593 return underflow(1) 

594 elif top_state.level() == new_state.level(): 

595 # assert len(stack) > 1, stack # == 

596 # This is the only difference between i.cut_stack and python/cs.cut_stack 

597 if len(stack) <= 1: 

598 return underflow(2) 

599 break 

600 else: 

601 # This happens often in valid Python programs. 

602 break 

603 # Restore the guard entry if necessary. 

604 if len(stack) == 1: 

605 stack.append(stack[-1]) 

606 elif len(stack) <= 1: 

607 return underflow(3) 

608 return None 

609 #@+node:ekr.20161108160409.3: *5* i.end_block 

610 def end_block(self, line, new_state, stack): 

611 # The block is ending. Add tail lines until the start of the next block. 

612 p = stack[-1].p 

613 self.add_line(p, line) 

614 self.cut_stack(new_state, stack) 

615 tail_p = None if self.gen_refs else p 

616 return tail_p 

617 #@+node:ekr.20161127102339.1: *5* i.ends_block 

618 def ends_block(self, line, new_state, prev_state, stack): 

619 """True if line ends the block.""" 

620 # Comparing new_state against prev_state does not work for python. 

621 top = stack[-1] 

622 return new_state.level() < top.state.level() 

623 #@+node:ekr.20161108160409.8: *5* i.gen_ref 

624 def gen_ref(self, line, parent, target): 

625 """ 

626 Generate the ref line. Return the headline. 

627 """ 

628 indent_ws = self.get_str_lws(line) 

629 h = self.clean_headline(line, p=None) 

630 if self.gen_refs: 

631 # Fix #441: Make sure all section refs are unique. 

632 d = self.refs_dict 

633 n = d.get(h, 0) 

634 d[h] = n + 1 

635 if n > 0: 

636 h = '%s: %s' % (n, h) 

637 headline = g.angleBrackets(' %s ' % h) 

638 ref = '%s%s\n' % ( 

639 indent_ws, 

640 g.angleBrackets(' %s ' % h)) 

641 else: 

642 if target.ref_flag: 

643 ref = None 

644 else: 

645 ref = '%s@others\n' % indent_ws 

646 target.at_others_flag = True 

647 target.ref_flag = True # Don't generate another @others in this target. 

648 headline = h 

649 if ref: 

650 self.add_line(parent, ref) 

651 return headline 

652 #@+node:ekr.20161108160409.6: *5* i.start_new_block 

653 def start_new_block(self, i, lines, new_state, prev_state, stack): 

654 """Create a child node and update the stack.""" 

655 if hasattr(new_state, 'in_context'): 

656 assert not new_state.in_context(), ('start_new_block', new_state) 

657 line = lines[i] 

658 target = stack[-1] 

659 # Insert the reference in *this* node. 

660 h = self.gen_ref(line, target.p, target) 

661 # Create a new child and associated target. 

662 child = self.create_child_node(target.p, line, h) 

663 stack.append(Target(child, new_state)) 

664 #@+node:ekr.20161119124217.1: *5* i.starts_block 

665 def starts_block(self, i, lines, new_state, prev_state): 

666 """True if the new state starts a block.""" 

667 return new_state.level() > prev_state.level() 

668 #@+node:ekr.20161119162451.1: *5* i.trace_status 

669 def trace_status(self, line, new_state, prev_state, stack, top): 

670 """Print everything important in the i.gen_lines loop.""" 

671 print('') 

672 try: 

673 g.trace(repr(line)) 

674 except Exception: 

675 g.trace(f" top.p: {g.toUnicode(top.p.h)}") 

676 # print('len(stack): %s' % len(stack)) 

677 print(' new_state: %s' % new_state) 

678 print('prev_state: %s' % prev_state) 

679 # print(' top.state: %s' % top.state) 

680 g.printList(stack) 

681 #@+node:ekr.20161108131153.13: *4* Stage 2: i.post_pass & helpers 

682 def post_pass(self, parent): 

683 """ 

684 Optional Stage 2 of the importer pipeline, consisting of zero or more 

685 substages. Each substage alters nodes in various ways. 

686 

687 Subclasses may freely override this method, **provided** that all 

688 substages use the API for setting body text. Changing p.b directly will 

689 cause asserts to fail later in i.finish(). 

690 """ 

691 self.clean_all_headlines(parent) 

692 if self.add_context: 

693 self.add_class_names(parent) 

694 self.clean_all_nodes(parent) 

695 self.unindent_all_nodes(parent) 

696 # 

697 # This sub-pass must follow unindent_all_nodes. 

698 self.promote_trailing_underindented_lines(parent) 

699 self.promote_last_lines(parent) 

700 # 

701 # This probably should be the last sub-pass. 

702 self.delete_all_empty_nodes(parent) 

703 #@+node:ekr.20180524130023.1: *5* i.add_class_names 

704 # Note: this method is never called for @clean trees. 

705 file_pattern = re.compile(r'^(([@])+(auto|clean|edit|file|nosent))') 

706 

707 def add_class_names(self, p): 

708 """ 

709 Add class names to headlines for all descendant nodes. 

710 

711 Called only when @bool add-context-to-headlines is True. 

712 """ 

713 if g.unitTesting: 

714 return # Don't changes the expected headlines. 

715 after, fn, class_name = None, None, None 

716 for p in p.self_and_subtree(): 

717 # Part 1: update the status. 

718 m = self.file_pattern.match(p.h) 

719 if m: 

720 prefix = m.group(1) 

721 fn = g.shortFileName(p.h[len(prefix) :].strip()) 

722 after, class_name = None, None 

723 continue 

724 if p.h.startswith('@path '): 

725 after, fn, class_name = None, None, None 

726 elif p.h.startswith('class '): 

727 class_name = p.h[5:].strip() 

728 if class_name: 

729 after = p.nodeAfterTree() 

730 continue 

731 elif p == after: 

732 after, class_name = None, None 

733 # Part 2: update the headline. 

734 if class_name: 

735 if not p.h.startswith(class_name): 

736 p.h = '%s.%s' % (class_name, p.h) 

737 elif fn and self.add_file_context: 

738 tag = ' (%s)' % fn 

739 if not p.h.endswith(tag): 

740 p.h += tag 

741 #@+node:ekr.20161110125940.1: *5* i.clean_all_headlines 

742 def clean_all_headlines(self, parent): 

743 """ 

744 Clean all headlines in parent's tree by calling the language-specific 

745 clean_headline method. 

746 """ 

747 for p in parent.subtree(): 

748 # Note: i.gen_ref calls clean_headline without knowing p. 

749 # As a result, the first argument is required. 

750 h = self.clean_headline(p.h, p=p) 

751 if h and h != p.h: 

752 p.h = h 

753 

754 #@+node:ekr.20161110130157.1: *5* i.clean_all_nodes 

755 def clean_all_nodes(self, parent): 

756 """Clean the nodes in parent's tree, in a language-dependent way.""" 

757 # i.clean_nodes does nothing. 

758 # Subclasses may override as desired. 

759 # See perl_i.clean_nodes for an example. 

760 self.clean_nodes(parent) 

761 #@+node:ekr.20161110130709.1: *5* i.delete_all_empty_nodes 

762 def delete_all_empty_nodes(self, parent): 

763 """ 

764 Delete nodes consisting of nothing but whitespace. 

765 Move the whitespace to the preceding node. 

766 """ 

767 c = self.c 

768 aList = [] 

769 for p in parent.subtree(): 

770 back = p.threadBack() 

771 if back and back.v != parent.v and back.v != self.root.v and not p.isCloned(): 

772 lines = self.get_lines(p) 

773 # Move the whitespace from p to back. 

774 if all(z.isspace() for z in lines): 

775 self.extend_lines(back, lines) 

776 # New in Leo 5.7: empty nodes may have children. 

777 if p.hasChildren(): 

778 # Don't delete p. 

779 p.h = 'organizer' 

780 self.get_lines(p) 

781 else: 

782 # Do delete p. 

783 aList.append(p.copy()) 

784 if aList: 

785 c.deletePositionsInList(aList) # Don't redraw. 

786 #@+node:ekr.20161222122914.1: *5* i.promote_last_lines 

787 def promote_last_lines(self, parent): 

788 """A placeholder for rust_i.promote_last_lines.""" 

789 #@+node:ekr.20161110131509.1: *5* i.promote_trailing_underindented_lines 

790 def promote_trailing_underindented_lines(self, parent): 

791 """ 

792 Promote all trailing underindent lines to the node's parent node, 

793 deleting one tab's worth of indentation. Typically, this will remove 

794 the underindent escape. 

795 """ 

796 pattern = self.escape_pattern # A compiled regex pattern 

797 for p in parent.subtree(): 

798 lines = self.get_lines(p) 

799 tail = [] 

800 while lines: 

801 line = lines[-1] 

802 m = pattern.match(line) 

803 if m: 

804 lines.pop() 

805 n_str = m.group(1) 

806 try: 

807 n = int(n_str) 

808 except ValueError: 

809 break 

810 if n == abs(self.tab_width): 

811 new_line = line[len(m.group(0)) :] 

812 tail.append(new_line) 

813 else: 

814 g.trace('unexpected unindent value', n) 

815 g.trace(line) 

816 # Fix #652 by restoring the line. 

817 new_line = line[len(m.group(0)) :].lstrip() 

818 lines.append(new_line) 

819 break 

820 else: 

821 break 

822 if tail: 

823 parent = p.parent() 

824 if parent.parent() == self.root: 

825 parent = parent.parent() 

826 self.set_lines(p, lines) 

827 self.extend_lines(parent, reversed(tail)) 

828 #@+node:ekr.20161110130337.1: *5* i.unindent_all_nodes 

829 def unindent_all_nodes(self, parent): 

830 """Unindent all nodes in parent's tree.""" 

831 for p in parent.subtree(): 

832 lines = self.get_lines(p) 

833 if all(z.isspace() for z in lines): 

834 # Somewhat dubious, but i.check covers for us. 

835 self.set_lines(p, []) 

836 else: 

837 self.set_lines(p, self.undent(p)) 

838 #@+node:ekr.20161111023249.1: *4* Stage 3: i.finish & helpers 

839 def finish(self, parent): 

840 """ 

841 Stage 3 (the last) stage of the importer pipeline. 

842 

843 Subclasses should never need to override this method. 

844 """ 

845 # Put directives at the end, so as not to interfere with shebang lines, etc. 

846 self.add_root_directives(parent) 

847 # 

848 # Finally, remove all v._import_list temporaries. 

849 self.finalize_ivars(parent) 

850 #@+node:ekr.20161108160409.5: *5* i.add_root_directives 

851 def add_root_directives(self, parent): 

852 """Return the proper directives for the root node p.""" 

853 table = [ 

854 '@language %s\n' % self.language, 

855 '@tabwidth %d\n' % self.tab_width, 

856 ] 

857 if self.parse_body: 

858 pass 

859 elif self.has_lines(parent): 

860 # Make sure the last line ends with a newline. 

861 lines = self.get_lines(parent) 

862 if lines: 

863 last_line = lines.pop() 

864 last_line = last_line.rstrip() + '\n' 

865 self.add_line(parent, last_line) 

866 self.extend_lines(parent, table) 

867 else: 

868 self.set_lines(parent, table) 

869 #@+node:ekr.20161110042020.1: *5* i.finalize_ivars 

870 def finalize_ivars(self, parent): 

871 """ 

872 Update the body text of all nodes in parent's tree using the injected 

873 v._import_lines lists. 

874 """ 

875 for p in parent.self_and_subtree(): 

876 v = p.v 

877 # Make sure that no code in x.post_pass has mistakenly set p.b. 

878 assert not v._bodyString, repr(v._bodyString) 

879 lines = self.get_lines(p) 

880 if lines and not lines[-1].endswith('\n'): 

881 lines[-1] += '\n' 

882 v._bodyString = g.toUnicode(''.join(lines), reportErrors=True) 

883 #@+node:ekr.20161108131153.3: *4* Stage 4: i.check & helpers 

884 def check(self, unused_s, parent): 

885 """True if perfect import checks pass.""" 

886 if g.app.suppressImportChecks: 

887 g.app.suppressImportChecks = False 

888 return True 

889 c = self.c 

890 sfn = g.shortFileName(self.root.h) 

891 s1 = g.toUnicode(self.file_s, self.encoding) 

892 s2 = self.trial_write() 

893 lines1, lines2 = g.splitLines(s1), g.splitLines(s2) 

894 if 0: # An excellent trace for debugging. 

895 g.trace(c.shortFileName()) 

896 g.printObj(lines1, tag='lines1') 

897 g.printObj(lines2, tag='lines2') 

898 if self.strict: 

899 # Ignore blank lines only. 

900 # Adding nodes may add blank lines. 

901 lines1 = self.strip_blank_lines(lines1) 

902 lines2 = self.strip_blank_lines(lines2) 

903 else: 

904 # Ignore blank lines and leading whitespace. 

905 # Importing may regularize whitespace, and that's good. 

906 lines1 = self.strip_all(lines1) 

907 lines2 = self.strip_all(lines2) 

908 # Forgive trailing whitespace problems in the last line. 

909 # This is not the same as clean_last_lines. 

910 if lines1 and lines2 and lines1 != lines2: 

911 lines1[-1] = lines1[-1].rstrip() + '\n' 

912 lines2[-1] = lines2[-1].rstrip() + '\n' 

913 # self.trace_lines(lines1, lines2, parent) 

914 ok = lines1 == lines2 

915 if not ok and not self.strict: 

916 # Issue an error only if something *other than* lws is amiss. 

917 lines1, lines2 = self.strip_lws(lines1), self.strip_lws(lines2) 

918 ok = lines1 == lines2 

919 if ok and not g.unitTesting: 

920 print('warning: leading whitespace changed in:', self.root.h) 

921 if not ok: 

922 self.show_failure(lines1, lines2, sfn) 

923 if g.unitTesting: 

924 assert False, 'Perfect import failed!' 

925 return ok 

926 #@+node:ekr.20161124030004.1: *5* i.clean_last_lines 

927 def clean_last_lines(self, lines): 

928 """Remove blank lines from the end of lines.""" 

929 while lines and lines[-1].isspace(): 

930 lines.pop() 

931 return lines 

932 #@+node:ekr.20170404035138.1: *5* i.context_lines 

933 def context_lines(self, aList, i, n=2): 

934 """Return a list containing the n lines of surrounding context of aList[i].""" 

935 result = [] 

936 aList1 = aList[max(0, i - n) : i] 

937 aList2 = aList[i + 1 : i + n + 1] 

938 result.extend([' %4s %r\n' % (i + 1 - len(aList1) + j, g.truncate(s, 60)) 

939 for j, s in enumerate(aList1)]) 

940 result.append('* %4s %r\n' % (i + 1, g.truncate(aList[i], 60))) 

941 result.extend([' %4s %r\n' % (i + 2 + j, g.truncate(s, 60)) 

942 for j, s in enumerate(aList2)]) 

943 return result 

944 #@+node:ekr.20161123210716.1: *5* i.show_failure 

945 def show_failure(self, lines1, lines2, sfn): 

946 """Print the failing lines, with surrounding context.""" 

947 if not g.unitTesting: 

948 g.es('@auto failed:', sfn, color='red') 

949 n1, n2 = len(lines1), len(lines2) 

950 print('\n===== PERFECT IMPORT FAILED =====', sfn) 

951 print('len(s1): %s len(s2): %s' % (n1, n2)) 

952 n_min = min(n1, n2) 

953 for i in range(n_min): 

954 line1, line2 = lines1[i], lines2[i] 

955 if line1 != line2: 

956 print('first mismatched line: %s' % (i + 1)) 

957 print('Expected...') 

958 print(''.join(self.context_lines(lines1, i))) 

959 print('Got...') 

960 print(''.join(self.context_lines(lines2, i))) 

961 break 

962 else: 

963 lines_s = 'n2' if n1 > n2 else 'n1' 

964 print(f"missing tail lines in {lines_s}") 

965 g.printObj(lines1, tag='lines1') 

966 g.printObj(lines2, tag='lines2') 

967 #@+node:ekr.20161108131153.5: *5* i.strip_* 

968 def lstrip_line(self, s): 

969 """Delete leading whitespace, *without* deleting the trailing newline!""" 

970 # This fixes a major bug in strip_lws. 

971 assert s, g.callers() 

972 return '\n' if s.isspace() else s.lstrip() 

973 

974 def strip_all(self, lines): 

975 """Strip blank lines and leading whitespace from all lines of s.""" 

976 return self.strip_lws(self.strip_blank_lines(lines)) 

977 

978 def strip_blank_lines(self, lines): 

979 """Strip all blank lines from s.""" 

980 return [z for z in lines if not z.isspace()] 

981 

982 def strip_lws(self, lines): 

983 """Strip leading whitespace from all lines.""" 

984 return [self.lstrip_line(z) for z in lines] 

985 # This also works, but I prefer the "extra" call to lstrip(). 

986 # return ['\n' if z.isspace() else z.lstrip() for z in lines]. 

987 

988 

989 #@+node:ekr.20161123210335.1: *5* i.trace_lines 

990 def trace_lines(self, lines1, lines2, parent): 

991 """Show both s1 and s2.""" 

992 print('===== s1: %s' % parent.h) 

993 for i, s in enumerate(lines1): 

994 g.pr('%3s %r' % (i + 1, s)) 

995 print('===== s2') 

996 for i, s in enumerate(lines2): 

997 g.pr('%3s %r' % (i + 1, s)) 

998 #@+node:ekr.20161108131153.6: *5* i.trial_write 

999 def trial_write(self): 

1000 """Return the trial write for self.root.""" 

1001 at = self.c.atFileCommands 

1002 # Leo 5.6: Allow apparent section refs for *all* languages. 

1003 ivar = 'allow_undefined_refs' 

1004 try: 

1005 setattr(at, ivar, True) 

1006 result = at.atAutoToString(self.root) 

1007 finally: 

1008 if hasattr(at, ivar): 

1009 delattr(at, ivar) 

1010 return g.toUnicode(result, self.encoding) 

1011 #@+node:ekr.20161108131153.15: *3* i.Utils 

1012 #@+node:ekr.20211118082436.1: *4* i.dump_tree 

1013 def dump_tree(self, root, tag=None): 

1014 """ 

1015 Like LeoUnitTest.dump_tree. 

1016 """ 

1017 d = self.vnode_info if hasattr(self, 'vnode_info') else {} 

1018 if tag: 

1019 print(tag) 

1020 for p in root.self_and_subtree(): 

1021 print('') 

1022 print('level:', p.level(), p.h) 

1023 lines = d[p.v]['lines'] if p.v in d else g.splitLines(p.v.b) 

1024 g.printObj(lines) 

1025 #@+node:ekr.20161114012522.1: *4* i.all_contexts 

1026 def all_contexts(self, table): 

1027 """ 

1028 Return a list of all contexts contained in the third column of the given table. 

1029 

1030 This is a support method for unit tests. 

1031 """ 

1032 contexts = set() 

1033 d = table 

1034 for key in d: 

1035 aList = d.get(key) 

1036 for data in aList: 

1037 if len(data) == 4: 

1038 # It's an out-of-context entry. 

1039 contexts.add(data[2]) 

1040 # Order must not matter, so sorting is ok. 

1041 return sorted(contexts) 

1042 #@+node:ekr.20161108131153.12: *4* i.insert_ignore_directive 

1043 def insert_ignore_directive(self, parent): 

1044 c = self.c 

1045 # Do *not* update the screen by setting p.b. 

1046 parent.v.b = parent.v.b.rstrip() + '\n@ignore\n' 

1047 if g.unitTesting: 

1048 pass 

1049 elif parent.isAnyAtFileNode() and not parent.isAtAutoNode(): 

1050 g.warning('inserting @ignore') 

1051 c.import_error_nodes.append(parent.h) 

1052 #@+node:ekr.20161108155143.4: *4* i.match 

1053 def match(self, s, i, pattern): 

1054 """Return True if the pattern matches at s[i:]""" 

1055 return s[i : i + len(pattern)] == pattern 

1056 #@+node:ekr.20161108131153.18: *4* i.Messages 

1057 def error(self, s): 

1058 """Issue an error and cause a unit test to fail.""" 

1059 self.errors += 1 

1060 self.importCommands.errors += 1 

1061 

1062 def report(self, message): 

1063 if self.strict: 

1064 self.error(message) 

1065 else: 

1066 self.warning(message) 

1067 

1068 def warning(self, s): 

1069 if not g.unitTesting: 

1070 g.warning('Warning:', s) 

1071 #@+node:ekr.20161109045619.1: *4* i.print_lines 

1072 def print_lines(self, lines): 

1073 """Print lines for debugging.""" 

1074 print('[') 

1075 for line in lines: 

1076 print(repr(line)) 

1077 print(']') 

1078 

1079 print_list = print_lines 

1080 #@+node:ekr.20161125174423.1: *4* i.print_stack 

1081 def print_stack(self, stack): 

1082 """Print a stack of positions.""" 

1083 g.printList([p.h for p in stack]) 

1084 #@+node:ekr.20161108131153.21: *4* i.underindented_comment/line 

1085 def underindented_comment(self, line): 

1086 if self.at_auto_warns_about_leading_whitespace: 

1087 self.warning( 

1088 'underindented python comments.\n' + 

1089 'Extra leading whitespace will be added\n' + line) 

1090 

1091 def underindented_line(self, line): 

1092 if self.warn_about_underindented_lines: 

1093 self.error( 

1094 'underindented line.\n' 

1095 'Extra leading whitespace will be added\n' + line) 

1096 #@+node:ekr.20161109045312.1: *3* i.Whitespace 

1097 #@+node:ekr.20161108155143.3: *4* i.get_int_lws 

1098 def get_int_lws(self, s): 

1099 """Return the the lws (a number) of line s.""" 

1100 # Important: use self.tab_width, *not* c.tab_width. 

1101 return g.computeLeadingWhitespaceWidth(s, self.tab_width) 

1102 #@+node:ekr.20161109053143.1: *4* i.get_leading_indent 

1103 def get_leading_indent(self, lines, i, ignoreComments=True): 

1104 """ 

1105 Return the leading whitespace (an int) of the first significant line. 

1106 Ignore blank and comment lines if ignoreComments is True 

1107 """ 

1108 if ignoreComments: 

1109 while i < len(lines): 

1110 if self.is_ws_line(lines[i]): 

1111 i += 1 

1112 else: 

1113 break 

1114 return self.get_int_lws(lines[i]) if i < len(lines) else 0 

1115 #@+node:ekr.20161108131153.17: *4* i.get_str_lws 

1116 def get_str_lws(self, s): 

1117 """Return the characters of the lws of s.""" 

1118 m = re.match(r'([ \t]*)', s) 

1119 return m.group(0) if m else '' 

1120 #@+node:ekr.20161109052011.1: *4* i.is_ws_line 

1121 def is_ws_line(self, s): 

1122 """Return True if s is nothing but whitespace and single-line comments.""" 

1123 return bool(self.ws_pattern.match(s)) 

1124 #@+node:ekr.20161108131153.19: *4* i.undent & helper 

1125 def undent(self, p): 

1126 """ 

1127 Remove the *maximum* whitespace of any line from the start of *all* lines, 

1128 appending the underindent escape sequence for all underindented lines. 

1129 

1130 This is *not* the same as textwrap.dedent! 

1131 

1132 """ 

1133 # Called from i.post_pass, i.unindent_all_nodes. 

1134 c = self.c 

1135 if self.is_rst: 

1136 return p.b # Never unindent rst code. 

1137 escape = c.atFileCommands.underindentEscapeString 

1138 lines = self.get_lines(p) 

1139 ws = self.common_lws(lines) 

1140 result = [] 

1141 for s in lines: 

1142 if s.startswith(ws): 

1143 result.append(s[len(ws) :]) 

1144 elif s.isspace(): 

1145 # Never change blank lines. 

1146 result.append(s) 

1147 else: 

1148 # Indicate that the line is underindented. 

1149 lws = g.get_leading_ws(s) 

1150 # Bug fix 2021/11/15: Use n1 - n2, not n1! 

1151 n1 = g.computeWidth(ws, self.tab_width) 

1152 n2 = g.computeWidth(lws, self.tab_width) 

1153 assert n1 > n2, (n1, n2) 

1154 result.append(f"{escape}{n1-n2}.{s.lstrip()}") 

1155 return result 

1156 #@+node:ekr.20161108131153.20: *5* i.common_lws 

1157 def common_lws(self, lines): 

1158 """Return the lws (a string) common to all lines.""" 

1159 if not lines: 

1160 return '' 

1161 lws = self.get_str_lws(lines[0]) 

1162 for s in lines: 

1163 if not self.is_ws_line(s): 

1164 lws2 = self.get_str_lws(s) 

1165 if lws2.startswith(lws): 

1166 pass 

1167 elif lws.startswith(lws2): 

1168 lws = lws2 

1169 else: 

1170 lws = '' # Nothing in common. 

1171 break 

1172 return lws 

1173 #@+node:ekr.20161109072221.1: *4* i.undent_body_lines & helper 

1174 def undent_body_lines(self, lines, ignoreComments=True): 

1175 """ 

1176 Remove the first line's leading indentation from all lines. 

1177 Return the resulting string. 

1178 """ 

1179 s = ''.join(lines) 

1180 if self.is_rst: 

1181 return s # Never unindent rst code. 

1182 # Calculate the amount to be removed from each line. 

1183 undent_val = self.get_leading_indent(lines, 0, ignoreComments=ignoreComments) 

1184 if undent_val == 0: 

1185 return s 

1186 result = self.undent_by(s, undent_val) 

1187 return result 

1188 #@+node:ekr.20161108180655.2: *5* i.undent_by 

1189 def undent_by(self, s, undent_val): 

1190 """ 

1191 Remove leading whitespace equivalent to undent_val from each line. 

1192 

1193 Strict languages: prepend the underindent escape for underindented lines. 

1194 """ 

1195 if self.is_rst: 

1196 return s # Never unindent rst code. 

1197 result = [] 

1198 for line in g.splitlines(s): 

1199 lws_s = self.get_str_lws(line) 

1200 lws = g.computeWidth(lws_s, self.tab_width) 

1201 # Add underindentEscapeString only for strict languages. 

1202 if self.strict and not line.isspace() and lws < undent_val: 

1203 # End the underindent count with a period to 

1204 # protect against lines that start with a digit! 

1205 result.append("%s%s.%s" % ( 

1206 self.escape, undent_val - lws, line.lstrip())) 

1207 else: 

1208 s = g.removeLeadingWhitespace(line, undent_val, self.tab_width) 

1209 result.append(s) 

1210 return ''.join(result) 

1211 #@-others 

1212 

1213 @classmethod 

1214 def do_import(cls): 

1215 def f(c, s, parent): 

1216 return cls(c.importCommands).run(s, parent) 

1217 return f 

1218#@+node:ekr.20161108171914.1: ** class ScanState 

1219class ScanState: 

1220 """ 

1221 The base class for classes representing the state of the line-oriented 

1222 scan. 

1223 """ 

1224 

1225 def __init__(self, d=None): 

1226 """ScanState ctor.""" 

1227 if d: 

1228 indent = d.get('indent') 

1229 prev = d.get('prev') 

1230 self.indent = indent # NOT prev.indent 

1231 self.bs_nl = prev.bs_nl 

1232 self.context = prev.context 

1233 self.curlies = prev.curlies 

1234 self.parens = prev.parens 

1235 self.squares = prev.squares 

1236 else: 

1237 self.bs_nl = False 

1238 self.context = '' 

1239 self.curlies = self.indent = self.parens = self.squares = 0 

1240 

1241 #@+others 

1242 #@+node:ekr.20161118043146.1: *3* ScanState.__repr__ 

1243 def __repr__(self): 

1244 """ScanState.__repr__""" 

1245 return 'ScanState context: %r curlies: %s' % ( 

1246 self.context, self.curlies) 

1247 #@+node:ekr.20161119115215.1: *3* ScanState.level 

1248 def level(self): 

1249 """ScanState.level.""" 

1250 return self.curlies 

1251 #@+node:ekr.20161118043530.1: *3* ScanState.update 

1252 def update(self, data): 

1253 """ 

1254 Update the state using the 6-tuple returned by i.scan_line. 

1255 Return i = data[1] 

1256 """ 

1257 context, i, delta_c, delta_p, delta_s, bs_nl = data 

1258 self.bs_nl = bs_nl 

1259 self.context = context 

1260 self.curlies += delta_c 

1261 self.parens += delta_p 

1262 self.squares += delta_s 

1263 return i 

1264 

1265 #@-others 

1266#@+node:ekr.20161108155158.1: ** class Target 

1267class Target: 

1268 """ 

1269 A class describing a target node p. 

1270 state is used to cut back the stack. 

1271 """ 

1272 

1273 def __init__(self, p, state): 

1274 """Target ctor.""" 

1275 self.at_others_flag = False # True: @others has been generated for this target. 

1276 self.p = p 

1277 self.gen_refs = False # Can be forced True. 

1278 self.ref_flag = False # True: @others or section reference should be generated. 

1279 self.state = state 

1280 

1281 def __repr__(self): 

1282 return 'Target: %s @others: %s refs: %s p: %s' % ( 

1283 self.state, 

1284 int(self.at_others_flag), 

1285 int(self.gen_refs), 

1286 g.shortFileName(self.p.h), 

1287 ) 

1288#@-others 

1289#@@language python 

1290#@@tabwidth -4 

1291#@@pagewidth 70 

1292#@-leo