Overhaul data management to get rid of __control__
authorJeremy Stanley <fungi@yuggoth.org>
Sun, 27 Aug 2017 17:07:45 +0000 (17:07 +0000)
committerJeremy Stanley <fungi@yuggoth.org>
Mon, 28 Aug 2017 04:53:30 +0000 (04:53 +0000)
Remap __control__ subkeys in data as follows:

    default_files -> .mudpy.filing.categories
    include_dirs -> _load
    include_files -> _load
    read_only -> _lock

Additionally, obsolete the __control__.private_files key by
switching from raw file/directory names to associative arrays in the
category defaults so that flags such as "private" can be added to
them, and make the category filenames automatically determined so
that they can be omitted unless specific overrides are required.

For the sake of future-proofing, rename the DataFile class to Data
and its "filename" attribute to "source" so that we avoid confusing
renames later when these might instead refer to some other storage
medium such as a table in a database. Similarly rename the Element
class attribute "filename" to "origin" attribute both to make it
more generic and to avoid confusion with "source" (an Element's
origin is a complete Data object, while a Data's source is just a
rooted-anchored file path currently).

Add a Universe.add_category convenience method to handle properly
guessing the corresponding fallback path and copying any declared
flags.

etc/mudpy.yaml
mudpy/data.py
mudpy/misc.py
mudpy/tests/fixtures/test_daemon.yaml
sample/__init__.yaml
share/archetype.yaml
share/command.yaml
share/menu.yaml

index 482666f..76d1f01 100644 (file)
@@ -2,13 +2,12 @@
 # to use, copy, modify, and distribute this software is granted under
 # terms provided in the LICENSE file distributed with this software.
 
 # to use, copy, modify, and distribute this software is granted under
 # terms provided in the LICENSE file distributed with this software.
 
-__control__:
-    default_files: { "account": "account.yaml", "actor": "actor.yaml", "area": "area.yaml", "command": "command.yaml", "internal": "internal.yaml", "menu": "menu.yaml", "other": "other.yaml", "prop": "prop.yaml" }
-    include_dirs: [ "sample" ]
-    include_files: [ "archetype.yaml" ]
-    private_files: [ "account.yaml" ]
-    read_only: yes
 
 
+_load: [ archetype.yaml, command.yaml, menu.yaml, sample ]
+
+_lock: true
+
+.mudpy.filing.categories: { account: { flags: [ private ] } }
 .mudpy.filing.prefix: "."
 .mudpy.filing.search: [ "", "etc", "share", "data" ]
 .mudpy.filing.stash: "data"
 .mudpy.filing.prefix: "."
 .mudpy.filing.search: [ "", "etc", "share", "data" ]
 .mudpy.filing.stash: "data"
index e9a1079..f113369 100644 (file)
@@ -12,24 +12,36 @@ import mudpy
 import yaml
 
 
 import yaml
 
 
-class DataFile:
+class Data:
 
     """A file containing universe elements and their facets."""
 
 
     """A file containing universe elements and their facets."""
 
-    def __init__(self, filename, universe):
-        self.filename = filename
+    def __init__(self,
+                 source,
+                 universe,
+                 flags=None,
+                 relative=None,
+                 ):
+        self.source = source
         self.universe = universe
         self.universe = universe
-        self.data = {}
+        if flags is None:
+            self.flags = []
+        else:
+            self.flags = flags[:]
+        self.relative = relative
         self.load()
 
     def load(self):
         """Read a file, create elements and poplulate facets accordingly."""
         self.modified = False
         self.load()
 
     def load(self):
         """Read a file, create elements and poplulate facets accordingly."""
         self.modified = False
+        self.source = find_file(
+                self.source, relative=self.relative, universe=self.universe)
         try:
         try:
-            self.data = yaml.safe_load(open(self.filename))
+            self.data = yaml.safe_load(open(self.source))
         except FileNotFoundError:
             # it's normal if the file is one which doesn't exist yet
         except FileNotFoundError:
             # it's normal if the file is one which doesn't exist yet
-            log_entry = ("File %s is unavailable." % self.filename, 6)
+            self.data = {}
+            log_entry = ("File %s is unavailable." % self.source, 6)
             try:
                 mudpy.misc.log(*log_entry)
             except NameError:
             try:
                 mudpy.misc.log(*log_entry)
             except NameError:
@@ -37,67 +49,27 @@ class DataFile:
                 self.universe.setup_loglines.append(log_entry)
         if not hasattr(self.universe, "files"):
             self.universe.files = {}
                 self.universe.setup_loglines.append(log_entry)
         if not hasattr(self.universe, "files"):
             self.universe.files = {}
-        self.universe.files[self.filename] = self
+        self.universe.files[self.source] = self
         includes = []
         includes = []
-        if "__control__" in self.data:
-            if "include_files" in self.data["__control__"]:
-                for included in self.data["__control__"]["include_files"]:
+        for node in list(self.data):
+            if node == "_load":
+                for included in self.data["_load"]:
                     included = find_file(
                         included,
                     included = find_file(
                         included,
-                        relative=self.filename,
+                        relative=self.source,
                         universe=self.universe)
                     if included not in includes:
                         includes.append(included)
                         universe=self.universe)
                     if included not in includes:
                         includes.append(included)
-            if "include_dirs" in self.data["__control__"]:
-                for included in [
-                    os.path.join(x, "__init__.yaml") for x in
-                        self.data["__control__"]["include_dirs"]
-                ]:
-                    included = find_file(
-                        included,
-                        relative=self.filename,
-                        universe=self.universe
-                    )
-                    if included not in includes:
-                        includes.append(included)
-            if "default_files" in self.data["__control__"]:
-                origins = self.data["__control__"]["default_files"]
-                for key in origins.keys():
-                    origins[key] = find_file(
-                        origins[key],
-                        relative=self.filename,
-                        universe=self.universe
-                    )
-                    if origins[key] not in includes:
-                        includes.append(origins[key])
-                    self.universe.default_origins[key] = origins[key]
-                    if key not in self.universe.categories:
-                        self.universe.categories[key] = {}
-            if "private_files" in self.data["__control__"]:
-                for item in self.data["__control__"]["private_files"]:
-                    item = find_file(
-                        item,
-                        relative=self.filename,
-                        universe=self.universe
-                    )
-                    if item not in includes:
-                        includes.append(item)
-                    if item not in self.universe.private_files:
-                        self.universe.private_files.append(item)
-        for node in list(self.data):
-            if node == "__control__":
                 continue
             facet_pos = node.rfind(".") + 1
             if not facet_pos:
                 continue
             facet_pos = node.rfind(".") + 1
             if not facet_pos:
-                mudpy.misc.Element(node, self.universe, self.filename,
-                                   old_style=True)
+                mudpy.misc.Element(node, self.universe, self, old_style=True)
             else:
                 prefix = node[:facet_pos].strip(".")
                 try:
                     element = self.universe.contents[prefix]
                 except KeyError:
             else:
                 prefix = node[:facet_pos].strip(".")
                 try:
                     element = self.universe.contents[prefix]
                 except KeyError:
-                    element = mudpy.misc.Element(prefix, self.universe,
-                                                 self.filename)
+                    element = mudpy.misc.Element(prefix, self.universe, self)
                 element.set(node[facet_pos:], self.data[node])
                 if prefix.startswith("mudpy.movement."):
                     self.universe.directions.add(
                 element.set(node[facet_pos:], self.data[node])
                 if prefix.startswith("mudpy.movement."):
                     self.universe.directions.add(
@@ -106,12 +78,12 @@ class DataFile:
             if not os.path.isabs(include_file):
                 include_file = find_file(
                     include_file,
             if not os.path.isabs(include_file):
                 include_file = find_file(
                     include_file,
-                    relative=self.filename,
+                    relative=self.source,
                     universe=self.universe
                 )
             if (include_file not in self.universe.files or not
                     self.universe.files[include_file].is_writeable()):
                     universe=self.universe
                 )
             if (include_file not in self.universe.files or not
                     self.universe.files[include_file].is_writeable()):
-                DataFile(include_file, self.universe)
+                Data(include_file, self.universe)
 
     def save(self):
         """Write the data, if necessary."""
 
     def save(self):
         """Write the data, if necessary."""
@@ -121,29 +93,26 @@ class DataFile:
 
         # when modified, writeable and has content or the file exists
         if self.modified and self.is_writeable() and (
 
         # when modified, writeable and has content or the file exists
         if self.modified and self.is_writeable() and (
-           self.data or os.path.exists(self.filename)
+           self.data or os.path.exists(self.source)
            ):
 
             # make parent directories if necessary
            ):
 
             # make parent directories if necessary
-            if not os.path.exists(os.path.dirname(self.filename)):
+            if not os.path.exists(os.path.dirname(self.source)):
                 old_umask = os.umask(normal_umask)
                 old_umask = os.umask(normal_umask)
-                os.makedirs(os.path.dirname(self.filename))
+                os.makedirs(os.path.dirname(self.source))
                 os.umask(old_umask)
 
             # backup the file
                 os.umask(old_umask)
 
             # backup the file
-            if "__control__" in self.data and "backup_count" in self.data[
-                    "__control__"]:
-                max_count = self.data["__control__"]["backup_count"]
-            elif "mudpy.limit" in self.universe.contents:
+            if "mudpy.limit" in self.universe.contents:
                 max_count = self.universe.contents["mudpy.limit"].get(
                     "backups", 0)
             else:
                 max_count = 0
                 max_count = self.universe.contents["mudpy.limit"].get(
                     "backups", 0)
             else:
                 max_count = 0
-            if os.path.exists(self.filename) and max_count:
+            if os.path.exists(self.source) and max_count:
                 backups = []
                 backups = []
-                for candidate in os.listdir(os.path.dirname(self.filename)):
+                for candidate in os.listdir(os.path.dirname(self.source)):
                     if re.match(
                     if re.match(
-                       os.path.basename(self.filename) +
+                       os.path.basename(self.source) +
                        r"""\.\d+$""", candidate
                        ):
                         backups.append(int(candidate.split(".")[-1]))
                        r"""\.\d+$""", candidate
                        ):
                         backups.append(int(candidate.split(".")[-1]))
@@ -151,28 +120,28 @@ class DataFile:
                 backups.reverse()
                 for old_backup in backups:
                     if old_backup >= max_count - 1:
                 backups.reverse()
                 for old_backup in backups:
                     if old_backup >= max_count - 1:
-                        os.remove(self.filename + "." + str(old_backup))
+                        os.remove(self.source + "." + str(old_backup))
                     elif not os.path.exists(
                     elif not os.path.exists(
-                        self.filename + "." + str(old_backup + 1)
+                        self.source + "." + str(old_backup + 1)
                     ):
                         os.rename(
                     ):
                         os.rename(
-                            self.filename + "." + str(old_backup),
-                            self.filename + "." + str(old_backup + 1)
+                            self.source + "." + str(old_backup),
+                            self.source + "." + str(old_backup + 1)
                         )
                         )
-                if not os.path.exists(self.filename + ".0"):
-                    os.rename(self.filename, self.filename + ".0")
+                if not os.path.exists(self.source + ".0"):
+                    os.rename(self.source, self.source + ".0")
 
             # our data file
 
             # our data file
-            if self.filename in self.universe.private_files:
+            if "private" in self.flags:
                 old_umask = os.umask(private_umask)
                 old_umask = os.umask(private_umask)
-                file_descriptor = open(self.filename, "w")
+                file_descriptor = open(self.source, "w")
                 if oct(stat.S_IMODE(os.stat(
                 if oct(stat.S_IMODE(os.stat(
-                        self.filename)[stat.ST_MODE])) != private_file_mode:
+                        self.source)[stat.ST_MODE])) != private_file_mode:
                     # if it's marked private, chmod it appropriately
                     # if it's marked private, chmod it appropriately
-                    os.chmod(self.filename, private_file_mode)
+                    os.chmod(self.source, private_file_mode)
             else:
                 old_umask = os.umask(normal_umask)
             else:
                 old_umask = os.umask(normal_umask)
-                file_descriptor = open(self.filename, "w")
+                file_descriptor = open(self.source, "w")
             os.umask(old_umask)
 
             # write and close the file
             os.umask(old_umask)
 
             # write and close the file
@@ -184,15 +153,16 @@ class DataFile:
             self.modified = False
 
     def is_writeable(self):
             self.modified = False
 
     def is_writeable(self):
-        """Returns True if the __control__ read_only is False."""
+        """Returns True if the _lock is False."""
         try:
         try:
-            return not self.data["__control__"].get("read_only", False)
+            return not self.data.get("_lock", False)
         except KeyError:
             return True
 
 
 def find_file(
     file_name=None,
         except KeyError:
             return True
 
 
 def find_file(
     file_name=None,
+    category=None,
     prefix=None,
     relative=None,
     search=None,
     prefix=None,
     relative=None,
     search=None,
@@ -281,9 +251,18 @@ def find_file(
     # start hunting for the file now
     for each_path in clean_search:
 
     # start hunting for the file now
     for each_path in clean_search:
 
+        # construct the candidate path
+        candidate = os.path.join(each_path, file_name)
+
         # if the file exists and is readable, we're done
         # if the file exists and is readable, we're done
-        if os.path.isfile(os.path.join(each_path, file_name)):
-            file_name = os.path.realpath(os.path.join(each_path, file_name))
+        if os.path.isfile(candidate):
+            file_name = os.path.realpath(candidate)
+            break
+
+        # if the path is a directory, look for an __init__ file
+        if os.path.isdir(candidate):
+            file_name = os.path.realpath(
+                    os.path.join(candidate, "__init__.yaml"))
             break
 
     # it didn't exist after all, so use the default path instead
             break
 
     # it didn't exist after all, so use the default path instead
index b0b7290..d43a7e2 100644 (file)
@@ -23,7 +23,7 @@ class Element:
 
     """An element of the universe."""
 
 
     """An element of the universe."""
 
-    def __init__(self, key, universe, filename=None, old_style=False):
+    def __init__(self, key, universe, origin=None, old_style=False):
         """Set up a new element."""
 
         # TODO(fungi): This can be removed after the transition is complete
         """Set up a new element."""
 
         # TODO(fungi): This can be removed after the transition is complete
@@ -62,21 +62,16 @@ class Element:
                 self.category = "other"
                 self.subkey = self.key
             if self.category not in self.universe.categories:
                 self.category = "other"
                 self.subkey = self.key
             if self.category not in self.universe.categories:
-                self.category = "other"
-                self.subkey = self.key
+                self.universe.categories[self.category] = {}
 
 
-            # get an appropriate filename for the origin
-            if not filename:
-                filename = self.universe.default_origins[self.category]
-            if not os.path.isabs(filename):
-                filename = os.path.abspath(filename)
-
-            # add the file if it doesn't exist yet
-            if filename not in self.universe.files:
-                mudpy.data.DataFile(filename, self.universe)
+            # get an appropriate origin
+            if not origin:
+                self.universe.add_category(self.category)
+                origin = self.universe.files[
+                        self.universe.origins[self.category]["fallback"]]
 
         # record or reset a pointer to the origin file
 
         # record or reset a pointer to the origin file
-        self.origin = self.universe.files[filename]
+        self.origin = self.universe.files[origin.source]
 
         # add a data section to the origin if necessary
         if self.key not in self.origin.data:
 
         # add a data section to the origin if necessary
         if self.key not in self.origin.data:
@@ -88,8 +83,7 @@ class Element:
 
     def reload(self):
         """Create a new element and replace this one."""
 
     def reload(self):
         """Create a new element and replace this one."""
-        Element(self.key, self.universe, self.origin.filename,
-                old_style=self.old_style)
+        Element(self.key, self.universe, self.origin, old_style=self.old_style)
         del(self)
 
     def destroy(self):
         del(self)
 
     def destroy(self):
@@ -351,11 +345,10 @@ class Universe:
         """Initialize the universe."""
         self.categories = {}
         self.contents = {}
         """Initialize the universe."""
         self.categories = {}
         self.contents = {}
-        self.default_origins = {}
         self.directions = set()
         self.loading = False
         self.loglines = []
         self.directions = set()
         self.loading = False
         self.loglines = []
-        self.private_files = []
+        self.origins = {}
         self.reload_flag = False
         self.setup_loglines = []
         self.startdir = os.getcwd()
         self.reload_flag = False
         self.setup_loglines = []
         self.startdir = os.getcwd()
@@ -404,11 +397,20 @@ class Universe:
                         del self.files[data_filename]
 
             # start loading from the initial file
                         del self.files[data_filename]
 
             # start loading from the initial file
-            mudpy.data.DataFile(self.filename, self)
+            mudpy.data.Data(self.filename, self)
+
+        # load default storage locations for categories
+        if hasattr(self, "contents") and "mudpy.filing" in self.contents:
+            self.origins.update(self.contents["mudpy.filing"].get(
+                "categories", {}))
+
+        # add some builtin categories we know we'll need
+        for category in ("account", "actor", "internal"):
+            self.add_category(category)
 
         # make a list of inactive avatars
         inactive_avatars = []
 
         # make a list of inactive avatars
         inactive_avatars = []
-        for account in self.categories["account"].values():
+        for account in self.categories.get("account", {}).values():
             for avatar in account.get("avatars"):
                 try:
                     inactive_avatars.append(self.contents[avatar])
             for avatar in account.get("avatars"):
                 try:
                     inactive_avatars.append(self.contents[avatar])
@@ -505,6 +507,19 @@ class Universe:
         """Convenience method to get the elapsed time counter."""
         return self.categories["internal"]["counters"].get("elapsed")
 
         """Convenience method to get the elapsed time counter."""
         return self.categories["internal"]["counters"].get("elapsed")
 
+    def add_category(self, category, fallback=None):
+        """Set up category tracking/metadata."""
+        if category not in self.origins:
+            self.origins[category] = {}
+        if not fallback:
+            fallback = mudpy.data.find_file(
+                    ".".join((category, "yaml")), universe=self)
+        if "fallback" not in self.origins[category]:
+            self.origins[category]["fallback"] = fallback
+        flags = self.origins[category].get("flags", None)
+        if fallback not in self.files:
+            mudpy.data.Data(fallback, self, flags=flags)
+
 
 class User:
 
 
 class User:
 
@@ -917,7 +932,7 @@ class User:
         counter = 0
         while "avatar:" + self.account.get("name") + ":" + str(
             counter
         counter = 0
         while "avatar:" + self.account.get("name") + ":" + str(
             counter
-        ) in universe.categories["actor"].keys():
+        ) in universe.categories.get("actor", {}).keys():
             counter += 1
         self.avatar = Element(
             "actor:avatar:" + self.account.get("name") + ":" + str(
             counter += 1
         self.avatar = Element(
             "actor:avatar:" + self.account.get("name") + ":" + str(
@@ -1375,10 +1390,8 @@ def on_pulse():
         user.pulse()
 
     # add an element for counters if it doesn't exist
         user.pulse()
 
     # add an element for counters if it doesn't exist
-    if "counters" not in universe.categories["internal"]:
-        universe.categories["internal"]["counters"] = Element(
-            "internal:counters", universe, old_style=True
-        )
+    if "counters" not in universe.categories.get("internal", {}):
+        Element("internal:counters", universe, old_style=True)
 
     # update the log every now and then
     if not universe.categories["internal"]["counters"].get("mark"):
 
     # update the log every now and then
     if not universe.categories["internal"]["counters"].get("mark"):
@@ -1704,7 +1717,7 @@ def handler_entering_account_name(user):
             user.error = "bad_name"
 
         # if that account exists, time to request a password
             user.error = "bad_name"
 
         # if that account exists, time to request a password
-        elif name in universe.categories["account"]:
+        elif name in universe.categories.get("account", {}):
             user.account = universe.categories["account"][name]
             user.state = "checking_password"
 
             user.account = universe.categories["account"][name]
             user.state = "checking_password"
 
@@ -2163,7 +2176,7 @@ def command_show(actor, parameters):
         elif arguments[1].strip(".") in universe.contents:
             element = universe.contents[arguments[1].strip(".")]
             message = ('These are the properties of the "' + arguments[1]
         elif arguments[1].strip(".") in universe.contents:
             element = universe.contents[arguments[1].strip(".")]
             message = ('These are the properties of the "' + arguments[1]
-                       + '" element (in "' + element.origin.filename
+                       + '" element (in "' + element.origin.source
                        + '"):$(eol)')
             facets = element.facets()
             for facet in sorted(facets):
                        + '"):$(eol)')
             facets = element.facets()
             for facet in sorted(facets):
@@ -2298,7 +2311,7 @@ def command_set(actor, parameters):
                     message = ('The "%s" element is kept in read-only file '
                                '"%s" and cannot be altered.' %
                                (element, universe.contents[
                     message = ('The "%s" element is kept in read-only file '
                                '"%s" and cannot be altered.' %
                                (element, universe.contents[
-                                        element].origin.filename))
+                                        element].origin.source))
                 except ValueError:
                     message = ('Value "%s" of type "%s" cannot be coerced '
                                'to the correct datatype for facet "%s".' %
                 except ValueError:
                     message = ('Value "%s" of type "%s" cannot be coerced '
                                'to the correct datatype for facet "%s".' %
index 65cc798..5d065e8 100644 (file)
@@ -2,13 +2,12 @@
 # to use, copy, modify, and distribute this software is granted under
 # terms provided in the LICENSE file distributed with this software.
 
 # to use, copy, modify, and distribute this software is granted under
 # terms provided in the LICENSE file distributed with this software.
 
-__control__:
-    default_files: { "account": "account.yaml", "actor": "actor.yaml", "area": "area.yaml", "command": "command.yaml", "internal": "internal.yaml", "menu": "menu.yaml", "other": "other.yaml", "prop": "prop.yaml" }
-    include_dirs: [ "sample" ]
-    include_files: [ "archetype.yaml" ]
-    private_files: [ "account.yaml" ]
-    read_only: yes
 
 
+_load: [ archetype.yaml, command.yaml, menu.yaml, sample ]
+
+_lock: true
+
+.mudpy.filing.categories: { account: { flags: [ private ] } }
 .mudpy.filing.prefix: "."
 .mudpy.filing.search: [ "", "etc", "share", "data" ]
 .mudpy.filing.stash: "data"
 .mudpy.filing.prefix: "."
 .mudpy.filing.search: [ "", "etc", "share", "data" ]
 .mudpy.filing.stash: "data"
index 97d2259..da51bad 100644 (file)
@@ -2,6 +2,6 @@
 # to use, copy, modify, and distribute this software is granted under
 # terms provided in the LICENSE file distributed with this software.
 
 # to use, copy, modify, and distribute this software is granted under
 # terms provided in the LICENSE file distributed with this software.
 
-__control__:
-    include_files: [ "area.yaml", "prop.yaml" ]
-    read_only: yes
+_load: [ area.yaml, prop.yaml ]
+
+_lock: true
index e1f87e6..b627080 100644 (file)
@@ -2,8 +2,7 @@
 # to use, copy, modify, and distribute this software is granted under
 # terms provided in the LICENSE file distributed with this software.
 
 # to use, copy, modify, and distribute this software is granted under
 # terms provided in the LICENSE file distributed with this software.
 
-__control__:
-    read_only: yes
+_lock: true
 
 archetype:actor:
     is_actor: yes
 
 archetype:actor:
     is_actor: yes
index d756839..fc4ce88 100644 (file)
@@ -2,8 +2,7 @@
 # to use, copy, modify, and distribute this software is granted under
 # terms provided in the LICENSE file distributed with this software.
 
 # to use, copy, modify, and distribute this software is granted under
 # terms provided in the LICENSE file distributed with this software.
 
-__control__:
-    read_only: yes
+_lock: true
 
 command:chat:
     action: command_chat(actor)
 
 command:chat:
     action: command_chat(actor)
index afc4808..968c5f2 100644 (file)
@@ -2,8 +2,7 @@
 # to use, copy, modify, and distribute this software is granted under
 # terms provided in the LICENSE file distributed with this software.
 
 # to use, copy, modify, and distribute this software is granted under
 # terms provided in the LICENSE file distributed with this software.
 
-__control__:
-    read_only: yes
+_lock: true
 
 menu:activate_avatar:
     action: user.activate_avatar_by_index(int(choice)-1)
 
 menu:activate_avatar:
     action: user.activate_avatar_by_index(int(choice)-1)