Coverage for C:\leo.repo\leo-editor\leo\core\leoPersistence.py: 51%

Shortcuts on this page

r m x   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

310 statements  

1# -*- coding: utf-8 -*- 

2#@+leo-ver=5-thin 

3#@+node:ekr.20140821055201.18331: * @file leoPersistence.py 

4#@@first 

5"""Support for persistent clones, gnx's and uA's using @persistence trees.""" 

6import binascii 

7import pickle 

8from leo.core import leoGlobals as g 

9#@+others 

10#@+node:ekr.20140711111623.17886: ** Commands (leoPersistence.py) 

11 

12@g.command('clean-persistence') 

13def view_pack_command(event): 

14 """Remove all @data nodes that do not correspond to an existing foreign file.""" 

15 c = event.get('c') 

16 if c and c.persistenceController: 

17 c.persistenceController.clean() 

18#@+node:ekr.20140711111623.17790: ** class PersistenceDataController 

19class PersistenceDataController: 

20 #@+<< docstring >> 

21 #@+node:ekr.20140711111623.17791: *3* << docstring >> (class persistenceController) 

22 """ 

23 A class to handle persistence in **foreign files**, that is, 

24 files created by @auto, @org-mode or @vim-outline node. 

25 

26 All required data are held in nodes having the following structure:: 

27 

28 - @persistence 

29 - @data <headline of foreign node> 

30 - @gnxs 

31 body text: pairs of lines: gnx:<gnx><newline>unl:<unl> 

32 - @uas 

33 @ua <gnx> 

34 body text: the pickled uA 

35 """ 

36 #@-<< docstring >> 

37 #@+others 

38 #@+node:ekr.20141023154408.3: *3* pd.ctor 

39 def __init__(self, c): 

40 """Ctor for persistenceController class.""" 

41 self.c = c 

42 self.at_persistence = None # The position of the @position node. 

43 #@+node:ekr.20140711111623.17793: *3* pd.Entry points 

44 #@+node:ekr.20140718153519.17731: *4* pd.clean 

45 def clean(self): 

46 """Remove all @data nodes that do not correspond to an existing foreign file.""" 

47 c = self.c 

48 at_persistence = self.has_at_persistence_node() 

49 if not at_persistence: 

50 return 

51 foreign_list = [ 

52 p.h.strip() for p in c.all_unique_positions() 

53 if self.is_foreign_file(p)] 

54 delete_list = [] 

55 tag = '@data:' 

56 for child in at_persistence.children(): 

57 if child.h.startswith(tag): 

58 name = child.h[len(tag) :].strip() 

59 if name not in foreign_list: 

60 delete_list.append(child.copy()) 

61 if delete_list: 

62 at_persistence.setDirty() 

63 c.setChanged() 

64 for p in delete_list: 

65 g.es_print('deleting:', p.h) 

66 c.deletePositionsInList(delete_list) 

67 c.redraw() 

68 #@+node:ekr.20140711111623.17804: *4* pd.update_before_write_foreign_file & helpers 

69 def update_before_write_foreign_file(self, root): 

70 """ 

71 Update the @data node for root, a foreign node. 

72 Create @gnxs nodes and @uas trees as needed. 

73 """ 

74 # Delete all children of the @data node. 

75 self.at_persistence = self.find_at_persistence_node() 

76 if not self.at_persistence: 

77 return None 

78 # was return at_data # for at-file-to-at-auto command. 

79 at_data = self.find_at_data_node(root) 

80 self.delete_at_data_children(at_data, root) 

81 # Create the data for the @gnxs and @uas trees. 

82 aList, seen = [], [] 

83 for p in root.subtree(): 

84 gnx = p.v.gnx 

85 assert gnx 

86 if gnx not in seen: 

87 seen.append(gnx) 

88 aList.append(p.copy()) 

89 # Create the @gnxs node 

90 at_gnxs = self.find_at_gnxs_node(root) 

91 at_gnxs.b = ''.join( 

92 [f"gnx: {p.v.gnx}\nunl: {self.relative_unl(p, root)}\n" 

93 for p in aList]) 

94 # Create the @uas tree. 

95 uas = [p for p in aList if p.v.u] 

96 if uas: 

97 at_uas = self.find_at_uas_node(root) 

98 if at_uas.hasChildren(): 

99 at_uas.v._deleteAllChildren() 

100 for p in uas: 

101 p2 = at_uas.insertAsLastChild() 

102 p2.h = '@ua:' + p.v.gnx 

103 p2.b = f"unl:{self.relative_unl(p, root)}\nua:{self.pickle(p)}" 

104 # This is no longer necessary because of at.saveOutlineIfPossible. 

105 # Explain why the .leo file has become dirty. 

106 # g.es_print(f"updated: @data:{root.h} ") 

107 return at_data # For at-file-to-at-auto command. 

108 #@+node:ekr.20140716021139.17773: *5* pd.delete_at_data_children 

109 def delete_at_data_children(self, at_data, root): 

110 """Delete all children of the @data node""" 

111 if at_data.hasChildren(): 

112 at_data.v._deleteAllChildren() 

113 #@+node:ekr.20140711111623.17807: *4* pd.update_after_read_foreign_file & helpers 

114 def update_after_read_foreign_file(self, root): 

115 """Restore gnx's, uAs and clone links using @gnxs nodes and @uas trees.""" 

116 self.at_persistence = self.find_at_persistence_node() 

117 if not self.at_persistence: 

118 return 

119 if not root: 

120 return 

121 if not self.is_foreign_file(root): 

122 return 

123 # Create clone links from @gnxs node 

124 at_gnxs = self.has_at_gnxs_node(root) 

125 if at_gnxs: 

126 self.restore_gnxs(at_gnxs, root) 

127 # Create uas from @uas tree. 

128 at_uas = self.has_at_uas_node(root) 

129 if at_uas: 

130 self.create_uas(at_uas, root) 

131 #@+node:ekr.20140711111623.17810: *5* pd.restore_gnxs & helpers 

132 def restore_gnxs(self, at_gnxs, root): 

133 """ 

134 Recreate gnx's and clone links from an @gnxs node. 

135 @gnxs nodes contain pairs of lines: 

136 gnx:<gnx> 

137 unl:<unl> 

138 """ 

139 lines = g.splitLines(at_gnxs.b) 

140 gnxs = [s[4:].strip() for s in lines if s.startswith('gnx:')] 

141 unls = [s[4:].strip() for s in lines if s.startswith('unl:')] 

142 if len(gnxs) == len(unls): 

143 d = self.create_outer_gnx_dict(root) 

144 for gnx, unl in zip(gnxs, unls): 

145 self.restore_gnx(d, gnx, root, unl) 

146 else: 

147 g.trace('bad @gnxs contents', gnxs, unls) 

148 #@+node:ekr.20141021083702.18341: *6* pd.create_outer_gnx_dict 

149 def create_outer_gnx_dict(self, root): 

150 """ 

151 Return a dict whose keys are gnx's and whose values are positions 

152 **outside** of root's tree. 

153 """ 

154 c, d = self.c, {} 

155 p = c.rootPosition() 

156 while p: 

157 if p.v == root.v: 

158 p.moveToNodeAfterTree() 

159 else: 

160 gnx = p.v.fileIndex 

161 d[gnx] = p.copy() 

162 p.moveToThreadNext() 

163 return d 

164 #@+node:ekr.20140711111623.17809: *6* pd.restore_gnx 

165 def restore_gnx(self, d, gnx, root, unl): 

166 """ 

167 d is an *outer* gnx dict, associating nodes *outside* the tree with positions. 

168 Let p1 be the position of the node *within* root's tree corresponding to unl. 

169 Let p2 be the position of any node *outside* root's tree with the given gnx. 

170 - Set p1.v.fileIndex = gnx. 

171 - If p2 exists, relink p1 so it is a clone of p2. 

172 """ 

173 p1 = self.find_position_for_relative_unl(root, unl) 

174 if not p1: 

175 return 

176 p2 = d.get(gnx) 

177 if p2: 

178 if p1.h == p2.h and p1.b == p2.b: 

179 p1._relinkAsCloneOf(p2) 

180 # Warning: p1 *no longer exists* here. 

181 # _relinkAsClone does *not* set p1.v = p2.v. 

182 else: 

183 g.es_print('mismatch in cloned node', p1.h) 

184 else: 

185 # Fix #526: A major bug: this was not set! 

186 p1.v.fileIndex = gnx 

187 g.app.nodeIndices.updateLastIndex(g.toUnicode(gnx)) 

188 #@+node:ekr.20140711111623.17892: *5* pd.create_uas 

189 def create_uas(self, at_uas, root): 

190 """Recreate uA's from the @ua nodes in the @uas tree.""" 

191 # Create an *inner* gnx dict. 

192 # Keys are gnx's, values are positions *within* root's tree. 

193 d = {} 

194 for p in root.self_and_subtree(copy=False): 

195 d[p.v.gnx] = p.copy() 

196 # Recreate the uA's for the gnx's given by each @ua node. 

197 for at_ua in at_uas.children(): 

198 h, b = at_ua.h, at_ua.b 

199 gnx = h[4:].strip() 

200 if b and gnx and g.match_word(h, 0, '@ua'): 

201 p = d.get(gnx) 

202 if p: 

203 # Handle all recent variants of the node. 

204 lines = g.splitLines(b) 

205 if b.startswith('unl:') and len(lines) == 2: 

206 # pylint: disable=unbalanced-tuple-unpacking 

207 unl, ua = lines 

208 else: 

209 unl, ua = None, b 

210 if ua.startswith('ua:'): 

211 ua = ua[3:] 

212 if ua: 

213 ua = self.unpickle(ua) 

214 p.v.u = ua 

215 else: 

216 g.trace('Can not unpickle uA in', 

217 p.h, repr(unl), type(ua), ua[:40]) 

218 #@+node:ekr.20140712105818.16750: *3* pd.Helpers 

219 #@+node:ekr.20140711111623.17845: *4* pd.at_data_body 

220 # Note: the unl of p relative to p is simply p.h, 

221 # so it is pointless to add that to @data nodes. 

222 

223 def at_data_body(self, p): 

224 """Return the body text for p's @data node.""" 

225 return f"gnx: {p.v.gnx}\n" 

226 #@+node:ekr.20140712105644.16744: *4* pd.expected_headline 

227 def expected_headline(self, p): 

228 """Return the expected imported headline for p.""" 

229 return getattr(p.v, '_imported_headline', p.h) 

230 #@+node:ekr.20140711111623.17854: *4* pd.find... 

231 # The find commands create the node if not found. 

232 #@+node:ekr.20140711111623.17856: *5* pd.find_at_data_node & helper 

233 def find_at_data_node(self, root): 

234 """ 

235 Return the @data node for root, a foreign node. 

236 Create the node if it does not exist. 

237 """ 

238 self.at_persistence = self.find_at_persistence_node() 

239 if not self.at_persistence: 

240 return None 

241 p = self.has_at_data_node(root) 

242 if p: 

243 return p 

244 p = self.at_persistence.insertAsLastChild() 

245 if not p: # #2103 

246 return None 

247 p.h = '@data:' + root.h 

248 p.b = self.at_data_body(root) 

249 return p 

250 #@+node:ekr.20140711111623.17857: *5* pd.find_at_gnxs_node 

251 def find_at_gnxs_node(self, root): 

252 """ 

253 Find the @gnxs node for root, a foreign node. 

254 Create the @gnxs node if it does not exist. 

255 """ 

256 h = '@gnxs' 

257 if not self.at_persistence: 

258 return None 

259 data = self.find_at_data_node(root) 

260 p = g.findNodeInTree(self.c, data, h) 

261 if p: 

262 return p 

263 p = data.insertAsLastChild() 

264 if p: # #2103 

265 p.h = h 

266 return p 

267 #@+node:ekr.20140711111623.17863: *5* pd.find_at_persistence_node 

268 def find_at_persistence_node(self): 

269 """ 

270 Find the first @persistence node in the outline. 

271 If it does not exist, create it as the *last* top-level node, 

272 so that no existing positions become invalid. 

273 """ 

274 c, h = self.c, '@persistence' 

275 p = g.findNodeAnywhere(c, h) 

276 if p: 

277 return p 

278 if c.config.getBool('create-at-persistence-nodes-automatically'): 

279 last = c.rootPosition() 

280 while last.hasNext(): 

281 last.moveToNext() 

282 p = last.insertAfter() 

283 if p: # #2103 

284 p.h = h 

285 g.es_print(f"created {h} node", color='red') 

286 return p 

287 #@+node:ekr.20140711111623.17891: *5* pd.find_at_uas_node 

288 def find_at_uas_node(self, root): 

289 """ 

290 Find the @uas node for root, a foreign node. 

291 Create the @uas node if it does not exist. 

292 """ 

293 h = '@uas' 

294 if not self.at_persistence: 

295 return None 

296 auto_view = self.find_at_data_node(root) 

297 p = g.findNodeInTree(self.c, auto_view, h) 

298 if p: 

299 return p 

300 p = auto_view.insertAsLastChild() 

301 if p: # #2103 

302 p.h = h 

303 return p 

304 #@+node:ekr.20140711111623.17861: *5* pd.find_position_for_relative_unl & helpers 

305 def find_position_for_relative_unl(self, root, unl): 

306 """ 

307 Given a unl relative to root, return the node whose 

308 unl matches the longest suffix of the given unl. 

309 """ 

310 unl_list = unl.split('-->') 

311 if not unl_list or len(unl_list) == 1 and not unl_list[0]: 

312 return root 

313 return self.find_exact_match(root, unl_list) 

314 # return self.find_best_match(root, unl_list) 

315 #@+node:ekr.20140716021139.17764: *6* pd.find_best_match 

316 def find_best_match(self, root, unl_list): 

317 """Find the best partial matches of the tail in root's tree.""" 

318 tail = unl_list[-1] 

319 matches = [] 

320 for p in root.self_and_subtree(copy=False): 

321 if p.h == tail: # A match 

322 # Compute the partial unl. 

323 parents = 0 

324 for parent2 in p.parents(): 

325 if parent2 == root: 

326 break 

327 elif parents + 2 > len(unl_list): 

328 break 

329 elif parent2.h != unl_list[-2 - parents]: 

330 break 

331 else: 

332 parents += 1 

333 matches.append((parents, p.copy()),) 

334 if matches: 

335 # Take the match with the greatest number of parents. 

336 

337 def key(aTuple): 

338 return aTuple[0] 

339 

340 n, p = list(sorted(matches, key=key))[-1] 

341 return p 

342 return None 

343 #@+node:ekr.20140716021139.17765: *6* pd.find_exact_match 

344 def find_exact_match(self, root, unl_list): 

345 """ 

346 Find an exact match of the unl_list in root's tree. 

347 The root does not appear in the unl_list. 

348 """ 

349 # full_unl = '-->'.join(unl_list) 

350 parent = root 

351 for unl in unl_list: 

352 for child in parent.children(): 

353 if child.h.strip() == unl.strip(): 

354 parent = child 

355 break 

356 else: 

357 return None 

358 return parent 

359 #@+node:ekr.20140711111623.17862: *5* pd.find_representative_node 

360 def find_representative_node(self, root, target): 

361 """ 

362 root is a foreign node. target is a gnxs node within root's tree. 

363 

364 Return a node *outside* of root's tree that is cloned to target, 

365 preferring nodes outside any @<file> tree. 

366 Never return any node in any @persistence tree. 

367 """ 

368 assert target 

369 assert root 

370 # Pass 1: accept only nodes outside any @file tree. 

371 p = self.c.rootPosition() 

372 while p: 

373 if p.h.startswith('@persistence'): 

374 p.moveToNodeAfterTree() 

375 elif p.isAnyAtFileNode(): 

376 p.moveToNodeAfterTree() 

377 elif p.v == target.v: 

378 return p 

379 else: 

380 p.moveToThreadNext() 

381 # Pass 2: accept any node outside the root tree. 

382 p = self.c.rootPosition() 

383 while p: 

384 if p.h.startswith('@persistence'): 

385 p.moveToNodeAfterTree() 

386 elif p == root: 

387 p.moveToNodeAfterTree() 

388 elif p.v == target.v: 

389 return p 

390 else: 

391 p.moveToThreadNext() 

392 g.trace('no representative node for:', target, 'parent:', target.parent()) 

393 return None 

394 #@+node:ekr.20140712105818.16751: *4* pd.foreign_file_name 

395 def foreign_file_name(self, p): 

396 """Return the file name for p, a foreign file node.""" 

397 for tag in ('@auto', '@org-mode', '@vim-outline'): 

398 if g.match_word(p.h, 0, tag): 

399 return p.h[len(tag) :].strip() 

400 return None 

401 #@+node:ekr.20140711111623.17864: *4* pd.has... 

402 # The has commands return None if the node does not exist. 

403 #@+node:ekr.20140711111623.17865: *5* pd.has_at_data_node 

404 def has_at_data_node(self, root): 

405 """ 

406 Return the @data node corresponding to root, a foreign node. 

407 Return None if no such node exists. 

408 """ 

409 # if g.unitTesting: 

410 # pass 

411 if not self.at_persistence: 

412 return None 

413 if not self.is_at_auto_node(root): 

414 return None 

415 # Find a direct child of the @persistence nodes with matching headline and body. 

416 s = self.at_data_body(root) 

417 for p in self.at_persistence.children(): 

418 if p.b == s: 

419 return p 

420 return None 

421 #@+node:ekr.20140711111623.17890: *5* pd.has_at_gnxs_node 

422 def has_at_gnxs_node(self, root): 

423 """ 

424 Find the @gnxs node for an @data node with the given unl. 

425 Return None if it does not exist. 

426 """ 

427 if self.at_persistence: 

428 p = self.has_at_data_node(root) 

429 return p and g.findNodeInTree(self.c, p, '@gnxs') 

430 return None 

431 #@+node:ekr.20140711111623.17894: *5* pd.has_at_uas_node 

432 def has_at_uas_node(self, root): 

433 """ 

434 Find the @uas node for an @data node with the given unl. 

435 Return None if it does not exist. 

436 """ 

437 if self.at_persistence: 

438 p = self.has_at_data_node(root) 

439 return p and g.findNodeInTree(self.c, p, '@uas') 

440 return None 

441 #@+node:ekr.20140711111623.17869: *5* pd.has_at_persistence_node 

442 def has_at_persistence_node(self): 

443 """Return the @persistence node or None if it does not exist.""" 

444 return g.findNodeAnywhere(self.c, '@persistence') 

445 #@+node:ekr.20140711111623.17870: *4* pd.is... 

446 #@+node:ekr.20140711111623.17871: *5* pd.is_at_auto_node 

447 def is_at_auto_node(self, p): 

448 """ 

449 Return True if p is *any* kind of @auto node, 

450 including @auto-otl and @auto-rst. 

451 """ 

452 return p.isAtAutoNode() 

453 # The safe way: it tracks changes to p.isAtAutoNode. 

454 #@+node:ekr.20140711111623.17897: *5* pd.is_at_file_node 

455 def is_at_file_node(self, p): 

456 """Return True if p is an @file node.""" 

457 return g.match_word(p.h, 0, '@file') 

458 #@+node:ekr.20140711111623.17872: *5* pd.is_cloned_outside_parent_tree 

459 def is_cloned_outside_parent_tree(self, p): 

460 """Return True if a clone of p exists outside the tree of p.parent().""" 

461 return len(list(set(p.v.parents))) > 1 

462 #@+node:ekr.20140712105644.16745: *5* pd.is_foreign_file 

463 def is_foreign_file(self, p): 

464 return ( 

465 self.is_at_auto_node(p) or 

466 g.match_word(p.h, 0, '@org-mode') or 

467 g.match_word(p.h, 0, '@vim-outline')) 

468 #@+node:ekr.20140713135856.17745: *4* pd.Pickling 

469 #@+node:ekr.20140713062552.17737: *5* pd.pickle 

470 def pickle(self, p): 

471 """Pickle val and return the hexlified result.""" 

472 try: 

473 ua = p.v.u 

474 s = pickle.dumps(ua, protocol=1) 

475 s2 = binascii.hexlify(s) 

476 s3 = g.toUnicode(s2, 'utf-8') 

477 return s3 

478 except pickle.PicklingError: 

479 g.warning("ignoring non-pickleable value", ua, "in", p.h) 

480 return '' 

481 except Exception: 

482 g.error("pd.pickle: unexpected exception in", p.h) 

483 g.es_exception() 

484 return '' 

485 #@+node:ekr.20140713135856.17744: *5* pd.unpickle 

486 def unpickle(self, s): 

487 """Unhexlify and unpickle string s into p.""" 

488 try: 

489 # Throws TypeError if s is not a hex string. 

490 bin = binascii.unhexlify(g.toEncodedString(s)) 

491 return pickle.loads(bin) 

492 except Exception: 

493 g.es_exception() 

494 return None 

495 #@+node:ekr.20140711111623.17879: *4* pd.unls... 

496 #@+node:ekr.20140711111623.17881: *5* pd.drop_unl_parent/tail 

497 def drop_unl_parent(self, unl): 

498 """Drop the penultimate part of the unl.""" 

499 aList = unl.split('-->') 

500 return '-->'.join(aList[:-2] + aList[-1:]) 

501 

502 def drop_unl_tail(self, unl): 

503 """Drop the last part of the unl.""" 

504 return '-->'.join(unl.split('-->')[:-1]) 

505 #@+node:ekr.20140711111623.17883: *5* pd.relative_unl 

506 def relative_unl(self, p, root): 

507 """Return the unl of p relative to the root position.""" 

508 result = [] 

509 for p in p.self_and_parents(copy=False): 

510 if p == root: 

511 break 

512 else: 

513 result.append(self.expected_headline(p)) 

514 return '-->'.join(reversed(result)) 

515 #@+node:ekr.20140711111623.17896: *5* pd.unl 

516 def unl(self, p): 

517 """Return the unl corresponding to the given position.""" 

518 return '-->'.join(reversed( 

519 [self.expected_headline(p2) for p2 in p.self_and_parents(copy=False)])) 

520 #@+node:ekr.20140711111623.17885: *5* pd.unl_tail 

521 def unl_tail(self, unl): 

522 """Return the last part of a unl.""" 

523 return unl.split('-->')[:-1][0] 

524 #@-others 

525#@-others 

526#@@language python 

527#@@tabwidth -4 

528#@@pagewidth 70 

529#@-leo