Coverage for C:\Repos\leo-editor\leo\core\leoPersistence.py: 51%
310 statements
« prev ^ index » next coverage.py v6.4, created at 2022-05-24 10:21 -0500
« prev ^ index » next coverage.py v6.4, created at 2022-05-24 10:21 -0500
1# -*- coding: utf-8 -*-
2#@+leo-ver=5-thin
3#@+node:ekr.20140821055201.18331: * @file leoPersistence.py
4#@@first
5"""Support for persistent clones, gnx's and uA's using @persistence trees."""
6import binascii
7import pickle
8from leo.core import leoGlobals as g
9#@+others
10#@+node:ekr.20140711111623.17886: ** Commands (leoPersistence.py)
12@g.command('clean-persistence')
13def view_pack_command(event):
14 """Remove all @data nodes that do not correspond to an existing foreign file."""
15 c = event.get('c')
16 if c and c.persistenceController:
17 c.persistenceController.clean()
18#@+node:ekr.20140711111623.17790: ** class PersistenceDataController
19class PersistenceDataController:
20 #@+<< docstring >>
21 #@+node:ekr.20140711111623.17791: *3* << docstring >> (class persistenceController)
22 """
23 A class to handle persistence in **foreign files**, that is,
24 files created by @auto, @org-mode or @vim-outline node.
26 All required data are held in nodes having the following structure::
28 - @persistence
29 - @data <headline of foreign node>
30 - @gnxs
31 body text: pairs of lines: gnx:<gnx><newline>unl:<unl>
32 - @uas
33 @ua <gnx>
34 body text: the pickled uA
35 """
36 #@-<< docstring >>
37 #@+others
38 #@+node:ekr.20141023154408.3: *3* pd.ctor
39 def __init__(self, c):
40 """Ctor for persistenceController class."""
41 self.c = c
42 self.at_persistence = None # The position of the @position node.
43 #@+node:ekr.20140711111623.17793: *3* pd.Entry points
44 #@+node:ekr.20140718153519.17731: *4* pd.clean
45 def clean(self):
46 """Remove all @data nodes that do not correspond to an existing foreign file."""
47 c = self.c
48 at_persistence = self.has_at_persistence_node()
49 if not at_persistence:
50 return
51 foreign_list = [
52 p.h.strip() for p in c.all_unique_positions()
53 if self.is_foreign_file(p)]
54 delete_list = []
55 tag = '@data:'
56 for child in at_persistence.children():
57 if child.h.startswith(tag):
58 name = child.h[len(tag) :].strip()
59 if name not in foreign_list:
60 delete_list.append(child.copy())
61 if delete_list:
62 at_persistence.setDirty()
63 c.setChanged()
64 for p in delete_list:
65 g.es_print('deleting:', p.h)
66 c.deletePositionsInList(delete_list)
67 c.redraw()
68 #@+node:ekr.20140711111623.17804: *4* pd.update_before_write_foreign_file & helpers
69 def update_before_write_foreign_file(self, root):
70 """
71 Update the @data node for root, a foreign node.
72 Create @gnxs nodes and @uas trees as needed.
73 """
74 # Delete all children of the @data node.
75 self.at_persistence = self.find_at_persistence_node()
76 if not self.at_persistence:
77 return None
78 # was return at_data # for at-file-to-at-auto command.
79 at_data = self.find_at_data_node(root)
80 self.delete_at_data_children(at_data, root)
81 # Create the data for the @gnxs and @uas trees.
82 aList, seen = [], []
83 for p in root.subtree():
84 gnx = p.v.gnx
85 assert gnx
86 if gnx not in seen:
87 seen.append(gnx)
88 aList.append(p.copy())
89 # Create the @gnxs node
90 at_gnxs = self.find_at_gnxs_node(root)
91 at_gnxs.b = ''.join(
92 [f"gnx: {p.v.gnx}\nunl: {self.relative_unl(p, root)}\n"
93 for p in aList])
94 # Create the @uas tree.
95 uas = [p for p in aList if p.v.u]
96 if uas:
97 at_uas = self.find_at_uas_node(root)
98 if at_uas.hasChildren():
99 at_uas.v._deleteAllChildren()
100 for p in uas:
101 p2 = at_uas.insertAsLastChild()
102 p2.h = '@ua:' + p.v.gnx
103 p2.b = f"unl:{self.relative_unl(p, root)}\nua:{self.pickle(p)}"
104 # This is no longer necessary because of at.saveOutlineIfPossible.
105 # Explain why the .leo file has become dirty.
106 # g.es_print(f"updated: @data:{root.h} ")
107 return at_data # For at-file-to-at-auto command.
108 #@+node:ekr.20140716021139.17773: *5* pd.delete_at_data_children
109 def delete_at_data_children(self, at_data, root):
110 """Delete all children of the @data node"""
111 if at_data.hasChildren():
112 at_data.v._deleteAllChildren()
113 #@+node:ekr.20140711111623.17807: *4* pd.update_after_read_foreign_file & helpers
114 def update_after_read_foreign_file(self, root):
115 """Restore gnx's, uAs and clone links using @gnxs nodes and @uas trees."""
116 self.at_persistence = self.find_at_persistence_node()
117 if not self.at_persistence:
118 return
119 if not root:
120 return
121 if not self.is_foreign_file(root):
122 return
123 # Create clone links from @gnxs node
124 at_gnxs = self.has_at_gnxs_node(root)
125 if at_gnxs:
126 self.restore_gnxs(at_gnxs, root)
127 # Create uas from @uas tree.
128 at_uas = self.has_at_uas_node(root)
129 if at_uas:
130 self.create_uas(at_uas, root)
131 #@+node:ekr.20140711111623.17810: *5* pd.restore_gnxs & helpers
132 def restore_gnxs(self, at_gnxs, root):
133 """
134 Recreate gnx's and clone links from an @gnxs node.
135 @gnxs nodes contain pairs of lines:
136 gnx:<gnx>
137 unl:<unl>
138 """
139 lines = g.splitLines(at_gnxs.b)
140 gnxs = [s[4:].strip() for s in lines if s.startswith('gnx:')]
141 unls = [s[4:].strip() for s in lines if s.startswith('unl:')]
142 if len(gnxs) == len(unls):
143 d = self.create_outer_gnx_dict(root)
144 for gnx, unl in zip(gnxs, unls):
145 self.restore_gnx(d, gnx, root, unl)
146 else:
147 g.trace('bad @gnxs contents', gnxs, unls)
148 #@+node:ekr.20141021083702.18341: *6* pd.create_outer_gnx_dict
149 def create_outer_gnx_dict(self, root):
150 """
151 Return a dict whose keys are gnx's and whose values are positions
152 **outside** of root's tree.
153 """
154 c, d = self.c, {}
155 p = c.rootPosition()
156 while p:
157 if p.v == root.v:
158 p.moveToNodeAfterTree()
159 else:
160 gnx = p.v.fileIndex
161 d[gnx] = p.copy()
162 p.moveToThreadNext()
163 return d
164 #@+node:ekr.20140711111623.17809: *6* pd.restore_gnx
165 def restore_gnx(self, d, gnx, root, unl):
166 """
167 d is an *outer* gnx dict, associating nodes *outside* the tree with positions.
168 Let p1 be the position of the node *within* root's tree corresponding to unl.
169 Let p2 be the position of any node *outside* root's tree with the given gnx.
170 - Set p1.v.fileIndex = gnx.
171 - If p2 exists, relink p1 so it is a clone of p2.
172 """
173 p1 = self.find_position_for_relative_unl(root, unl)
174 if not p1:
175 return
176 p2 = d.get(gnx)
177 if p2:
178 if p1.h == p2.h and p1.b == p2.b:
179 p1._relinkAsCloneOf(p2)
180 # Warning: p1 *no longer exists* here.
181 # _relinkAsClone does *not* set p1.v = p2.v.
182 else:
183 g.es_print('mismatch in cloned node', p1.h)
184 else:
185 # Fix #526: A major bug: this was not set!
186 p1.v.fileIndex = gnx
187 g.app.nodeIndices.updateLastIndex(g.toUnicode(gnx))
188 #@+node:ekr.20140711111623.17892: *5* pd.create_uas
189 def create_uas(self, at_uas, root):
190 """Recreate uA's from the @ua nodes in the @uas tree."""
191 # Create an *inner* gnx dict.
192 # Keys are gnx's, values are positions *within* root's tree.
193 d = {}
194 for p in root.self_and_subtree(copy=False):
195 d[p.v.gnx] = p.copy()
196 # Recreate the uA's for the gnx's given by each @ua node.
197 for at_ua in at_uas.children():
198 h, b = at_ua.h, at_ua.b
199 gnx = h[4:].strip()
200 if b and gnx and g.match_word(h, 0, '@ua'):
201 p = d.get(gnx)
202 if p:
203 # Handle all recent variants of the node.
204 lines = g.splitLines(b)
205 if b.startswith('unl:') and len(lines) == 2:
206 # pylint: disable=unbalanced-tuple-unpacking
207 unl, ua = lines
208 else:
209 unl, ua = None, b
210 if ua.startswith('ua:'):
211 ua = ua[3:]
212 if ua:
213 ua = self.unpickle(ua)
214 p.v.u = ua
215 else:
216 g.trace('Can not unpickle uA in',
217 p.h, repr(unl), type(ua), ua[:40])
218 #@+node:ekr.20140712105818.16750: *3* pd.Helpers
219 #@+node:ekr.20140711111623.17845: *4* pd.at_data_body
220 # Note: the unl of p relative to p is simply p.h,
221 # so it is pointless to add that to @data nodes.
223 def at_data_body(self, p):
224 """Return the body text for p's @data node."""
225 return f"gnx: {p.v.gnx}\n"
226 #@+node:ekr.20140712105644.16744: *4* pd.expected_headline
227 def expected_headline(self, p):
228 """Return the expected imported headline for p."""
229 return getattr(p.v, '_imported_headline', p.h)
230 #@+node:ekr.20140711111623.17854: *4* pd.find...
231 # The find commands create the node if not found.
232 #@+node:ekr.20140711111623.17856: *5* pd.find_at_data_node & helper
233 def find_at_data_node(self, root):
234 """
235 Return the @data node for root, a foreign node.
236 Create the node if it does not exist.
237 """
238 self.at_persistence = self.find_at_persistence_node()
239 if not self.at_persistence:
240 return None
241 p = self.has_at_data_node(root)
242 if p:
243 return p
244 p = self.at_persistence.insertAsLastChild()
245 if not p: # #2103
246 return None
247 p.h = '@data:' + root.h
248 p.b = self.at_data_body(root)
249 return p
250 #@+node:ekr.20140711111623.17857: *5* pd.find_at_gnxs_node
251 def find_at_gnxs_node(self, root):
252 """
253 Find the @gnxs node for root, a foreign node.
254 Create the @gnxs node if it does not exist.
255 """
256 h = '@gnxs'
257 if not self.at_persistence:
258 return None
259 data = self.find_at_data_node(root)
260 p = g.findNodeInTree(self.c, data, h)
261 if p:
262 return p
263 p = data.insertAsLastChild()
264 if p: # #2103
265 p.h = h
266 return p
267 #@+node:ekr.20140711111623.17863: *5* pd.find_at_persistence_node
268 def find_at_persistence_node(self):
269 """
270 Find the first @persistence node in the outline.
271 If it does not exist, create it as the *last* top-level node,
272 so that no existing positions become invalid.
273 """
274 c, h = self.c, '@persistence'
275 p = g.findNodeAnywhere(c, h)
276 if p:
277 return p
278 if c.config.getBool('create-at-persistence-nodes-automatically'):
279 last = c.rootPosition()
280 while last.hasNext():
281 last.moveToNext()
282 p = last.insertAfter()
283 if p: # #2103
284 p.h = h
285 g.es_print(f"created {h} node", color='red')
286 return p
287 #@+node:ekr.20140711111623.17891: *5* pd.find_at_uas_node
288 def find_at_uas_node(self, root):
289 """
290 Find the @uas node for root, a foreign node.
291 Create the @uas node if it does not exist.
292 """
293 h = '@uas'
294 if not self.at_persistence:
295 return None
296 auto_view = self.find_at_data_node(root)
297 p = g.findNodeInTree(self.c, auto_view, h)
298 if p:
299 return p
300 p = auto_view.insertAsLastChild()
301 if p: # #2103
302 p.h = h
303 return p
304 #@+node:ekr.20140711111623.17861: *5* pd.find_position_for_relative_unl & helpers
305 def find_position_for_relative_unl(self, root, unl):
306 """
307 Given a unl relative to root, return the node whose
308 unl matches the longest suffix of the given unl.
309 """
310 unl_list = unl.split('-->')
311 if not unl_list or len(unl_list) == 1 and not unl_list[0]:
312 return root
313 # return self.find_best_match(root, unl_list)
314 return self.find_exact_match(root, unl_list)
315 #@+node:ekr.20140716021139.17764: *6* pd.find_best_match
316 def find_best_match(self, root, unl_list):
317 """Find the best partial matches of the tail in root's tree."""
318 tail = unl_list[-1]
319 matches = []
320 for p in root.self_and_subtree(copy=False):
321 if p.h == tail: # A match
322 # Compute the partial unl.
323 parents = 0
324 for parent2 in p.parents():
325 if parent2 == root:
326 break
327 elif parents + 2 > len(unl_list):
328 break
329 elif parent2.h != unl_list[-2 - parents]:
330 break
331 else:
332 parents += 1
333 matches.append((parents, p.copy()),)
334 if matches:
335 # Take the match with the greatest number of parents.
337 def key(aTuple):
338 return aTuple[0]
340 n, p = list(sorted(matches, key=key))[-1]
341 return p
342 return None
343 #@+node:ekr.20140716021139.17765: *6* pd.find_exact_match
344 def find_exact_match(self, root, unl_list):
345 """
346 Find an exact match of the unl_list in root's tree.
347 The root does not appear in the unl_list.
348 """
349 # full_unl = '-->'.join(unl_list)
350 parent = root
351 for unl in unl_list:
352 for child in parent.children():
353 if child.h.strip() == unl.strip():
354 parent = child
355 break
356 else:
357 return None
358 return parent
359 #@+node:ekr.20140711111623.17862: *5* pd.find_representative_node
360 def find_representative_node(self, root, target):
361 """
362 root is a foreign node. target is a gnxs node within root's tree.
364 Return a node *outside* of root's tree that is cloned to target,
365 preferring nodes outside any @<file> tree.
366 Never return any node in any @persistence tree.
367 """
368 assert target
369 assert root
370 # Pass 1: accept only nodes outside any @file tree.
371 p = self.c.rootPosition()
372 while p:
373 if p.h.startswith('@persistence'):
374 p.moveToNodeAfterTree()
375 elif p.isAnyAtFileNode():
376 p.moveToNodeAfterTree()
377 elif p.v == target.v:
378 return p
379 else:
380 p.moveToThreadNext()
381 # Pass 2: accept any node outside the root tree.
382 p = self.c.rootPosition()
383 while p:
384 if p.h.startswith('@persistence'):
385 p.moveToNodeAfterTree()
386 elif p == root:
387 p.moveToNodeAfterTree()
388 elif p.v == target.v:
389 return p
390 else:
391 p.moveToThreadNext()
392 g.trace('no representative node for:', target, 'parent:', target.parent())
393 return None
394 #@+node:ekr.20140712105818.16751: *4* pd.foreign_file_name
395 def foreign_file_name(self, p):
396 """Return the file name for p, a foreign file node."""
397 for tag in ('@auto', '@org-mode', '@vim-outline'):
398 if g.match_word(p.h, 0, tag):
399 return p.h[len(tag) :].strip()
400 return None
401 #@+node:ekr.20140711111623.17864: *4* pd.has...
402 # The has commands return None if the node does not exist.
403 #@+node:ekr.20140711111623.17865: *5* pd.has_at_data_node
404 def has_at_data_node(self, root):
405 """
406 Return the @data node corresponding to root, a foreign node.
407 Return None if no such node exists.
408 """
409 # if g.unitTesting:
410 # pass
411 if not self.at_persistence:
412 return None
413 if not self.is_at_auto_node(root):
414 return None
415 # Find a direct child of the @persistence nodes with matching headline and body.
416 s = self.at_data_body(root)
417 for p in self.at_persistence.children():
418 if p.b == s:
419 return p
420 return None
421 #@+node:ekr.20140711111623.17890: *5* pd.has_at_gnxs_node
422 def has_at_gnxs_node(self, root):
423 """
424 Find the @gnxs node for an @data node with the given unl.
425 Return None if it does not exist.
426 """
427 if self.at_persistence:
428 p = self.has_at_data_node(root)
429 return p and g.findNodeInTree(self.c, p, '@gnxs')
430 return None
431 #@+node:ekr.20140711111623.17894: *5* pd.has_at_uas_node
432 def has_at_uas_node(self, root):
433 """
434 Find the @uas node for an @data node with the given unl.
435 Return None if it does not exist.
436 """
437 if self.at_persistence:
438 p = self.has_at_data_node(root)
439 return p and g.findNodeInTree(self.c, p, '@uas')
440 return None
441 #@+node:ekr.20140711111623.17869: *5* pd.has_at_persistence_node
442 def has_at_persistence_node(self):
443 """Return the @persistence node or None if it does not exist."""
444 return g.findNodeAnywhere(self.c, '@persistence')
445 #@+node:ekr.20140711111623.17870: *4* pd.is...
446 #@+node:ekr.20140711111623.17871: *5* pd.is_at_auto_node
447 def is_at_auto_node(self, p):
448 """
449 Return True if p is *any* kind of @auto node,
450 including @auto-otl and @auto-rst.
451 """
452 # The safe way: it tracks changes to p.isAtAutoNode.
453 return p.isAtAutoNode()
454 #@+node:ekr.20140711111623.17897: *5* pd.is_at_file_node
455 def is_at_file_node(self, p):
456 """Return True if p is an @file node."""
457 return g.match_word(p.h, 0, '@file')
458 #@+node:ekr.20140711111623.17872: *5* pd.is_cloned_outside_parent_tree
459 def is_cloned_outside_parent_tree(self, p):
460 """Return True if a clone of p exists outside the tree of p.parent()."""
461 return len(list(set(p.v.parents))) > 1
462 #@+node:ekr.20140712105644.16745: *5* pd.is_foreign_file
463 def is_foreign_file(self, p):
464 return (
465 self.is_at_auto_node(p) or
466 g.match_word(p.h, 0, '@org-mode') or
467 g.match_word(p.h, 0, '@vim-outline'))
468 #@+node:ekr.20140713135856.17745: *4* pd.Pickling
469 #@+node:ekr.20140713062552.17737: *5* pd.pickle
470 def pickle(self, p):
471 """Pickle val and return the hexlified result."""
472 try:
473 ua = p.v.u
474 s = pickle.dumps(ua, protocol=1)
475 s2 = binascii.hexlify(s)
476 s3 = g.toUnicode(s2, 'utf-8')
477 return s3
478 except pickle.PicklingError:
479 g.warning("ignoring non-pickleable value", ua, "in", p.h)
480 return ''
481 except Exception:
482 g.error("pd.pickle: unexpected exception in", p.h)
483 g.es_exception()
484 return ''
485 #@+node:ekr.20140713135856.17744: *5* pd.unpickle
486 def unpickle(self, s):
487 """Unhexlify and unpickle string s into p."""
488 try:
489 # Throws TypeError if s is not a hex string.
490 bin = binascii.unhexlify(g.toEncodedString(s))
491 return pickle.loads(bin)
492 except Exception:
493 g.es_exception()
494 return None
495 #@+node:ekr.20140711111623.17879: *4* pd.unls...
496 #@+node:ekr.20140711111623.17881: *5* pd.drop_unl_parent/tail
497 def drop_unl_parent(self, unl):
498 """Drop the penultimate part of the unl."""
499 aList = unl.split('-->')
500 return '-->'.join(aList[:-2] + aList[-1:])
502 def drop_unl_tail(self, unl):
503 """Drop the last part of the unl."""
504 return '-->'.join(unl.split('-->')[:-1])
505 #@+node:ekr.20140711111623.17883: *5* pd.relative_unl
506 def relative_unl(self, p, root):
507 """Return the unl of p relative to the root position."""
508 result = []
509 for p in p.self_and_parents(copy=False):
510 if p == root:
511 break
512 else:
513 result.append(self.expected_headline(p))
514 return '-->'.join(reversed(result))
515 #@+node:ekr.20140711111623.17896: *5* pd.unl
516 def unl(self, p):
517 """Return the unl corresponding to the given position."""
518 return '-->'.join(reversed(
519 [self.expected_headline(p2) for p2 in p.self_and_parents(copy=False)]))
520 #@+node:ekr.20140711111623.17885: *5* pd.unl_tail
521 def unl_tail(self, unl):
522 """Return the last part of a unl."""
523 return unl.split('-->')[:-1][0]
524 #@-others
525#@-others
526#@@language python
527#@@tabwidth -4
528#@@pagewidth 70
529#@-leo