Extended attributes for instruments

papers/newIns.md

@@ -159,6 +159,7 @@ the following feature codes are recognized:
 - `PN`: PowerNoise ins data
 - `S2`: SID2 ins data
 - `S3`: SID3 ins data
+- `XA`: extended attributes
 - `EN`: end of features
   - if you find this feature code, stop reading the instrument.
   - it will usually appear only when there are sample/wave lists.
@@ -770,3 +771,26 @@ size | description
   1  | resonance scaling level
   1  | resonance scaling center note: `0` is `c_5`, `1` is `c+5`, ..., `179` is `B-9` 
 ```
+
+# extended attributes (XA)
+
+for each attribute, up until a empty string (null byte only) is encountered for an attribute's name.
+```
+size | description
+-----|------------------------------------
+ STR | name
+  1  | attribute type
+     | - 0: string attribute
+     | - 1: unsigned integer attribute
+     | - 2: integer attribute
+     | - 3: floating-point attribute
+     | - 4: boolean attribute (false)
+     | - 5: boolean attribute (true)
+```
+
+if the attribute is a string, then a string is followed afterwards.
+if the attribute is an unsigned integer, the integer value encoded in LEB128 is followed afterwards.
+however, if the attribute is a integer, the first byte will only have 6 bits of data, the 7th bit in the first byte will be used as the sign bit of the whole integer.
+if the 7th bit in the first byte is on, the integer should be negated after decoding.
+if the attribute is a float, a 32-bit floating-point integer is followed afterwards.
+boolean attributes have their value encoded in the type itself.

src/engine/instrument.cpp

@@ -22,6 +22,7 @@
 #include "instrument.h"
 #include "../ta-log.h"
 #include "../fileutils.h"
+#include "../hashUtils.h"
 
 const DivInstrument defaultIns;
 
@@ -480,9 +481,22 @@ void DivInstrumentUndoStep::applyAndReverse(DivInstrument* target) {
   if (nameValid) {
     name.swap(target->name);
   }
+
+  if (xattrsValid) {
+    target->xattrs=xattrs;
+  }
   podPatch.applyAndReverse((DivInstrumentPOD*)target, sizeof(DivInstrumentPOD));
 }
 
+static size_t computeXattrsHash(const std::vector<DivInstrumentXattr> *vec) {
+  size_t hash=0;
+  std::hash<DivInstrumentXattr> hasher;
+  for (const auto &i: *vec) {
+    hash=combineHash(hash, hasher(i));
+  }
+
+  return hash;
+}
 bool DivInstrumentUndoStep::makeUndoPatch(size_t processTime_, const DivInstrument* pre, const DivInstrument* post) {
   processTime=processTime_;
 
@@ -493,7 +507,22 @@ bool DivInstrumentUndoStep::makeUndoPatch(size_t processTime_, const DivInstrume
     name=pre->name;
   }
 
-  return nameValid || podPatch.isValid();
+  // check xattrs
+  size_t hash=0;
+  size_t post_hash=0;
+  if (!pre->xattrs.empty() || !post->xattrs.empty()) {
+    hash=computeXattrsHash(&pre->xattrs);
+    post_hash=computeXattrsHash(&post->xattrs);
+    logD("Xattrs pre hash: %lld", hash);
+    logD("Xattrs post hash: %lld", post_hash);
+
+    if (hash!=post_hash) {
+      xattrsValid=true;
+      xattrs=pre->xattrs;
+    }
+  }
+
+  return xattrsValid || nameValid || podPatch.isValid();
 }
 
 bool DivInstrument::recordUndoStepIfChanged(size_t processTime, const DivInstrument* old) {
@@ -1131,6 +1160,69 @@ void DivInstrument::writeFeatureS3(SafeWriter* w) {
   FEATURE_END;
 }
 
+void DivInstrument::writeFeatureXA(SafeWriter* w) {
+  FEATURE_BEGIN("XA");
+  for (auto const &i : this->xattrs) {
+    w->writeString(i.name, false);
+    if ((i.type & ~1) == DIV_XATTR_BOOLEAN) {
+      // handle boolean data type here
+      // the unnecessary casting here is because of MSVC
+      w->writeC(i.type | (unsigned char)i.bool_val);
+      continue;
+    }
+    w->writeC(i.type);
+    switch (i.type) {
+      case DIV_XATTR_STRING:
+      w->writeString(i.str_val, false);
+      break;
+      case DIV_XATTR_UINT: {
+        unsigned int value = i.uint_val;
+        unsigned char vlq_byte = 0;
+
+        do {
+          vlq_byte = value & 0x7F;
+          value >>= 7;
+          vlq_byte |= (value != 0) << 7;
+          w->writeC(vlq_byte);
+        } while (value);
+
+        break;
+      }
+      case DIV_XATTR_INT: {
+        int value = i.int_val;
+        unsigned char vlq_byte = 0;
+
+        // write initial VLQ byte (which contains sign bit)
+        if (i.int_val < 0) {
+          value = -value;
+          vlq_byte |= 0x40;
+        }
+
+        vlq_byte |= value & 0x3F;
+        value >>= 6;
+        vlq_byte |= (value != 0) << 7;
+        w->writeC(vlq_byte);
+
+        while (value) {
+          vlq_byte = value & 0x7F;
+          value >>= 7;
+          vlq_byte |= (value != 0) << 7;
+          w->writeC(vlq_byte);
+        }
+        break;
+      }
+      case DIV_XATTR_FLOAT32:
+      w->writeF(i.float_val);
+      break;
+      default:
+      // this should be unreachable
+      break;
+    }
+  }
+  // a empty string, as feature termination
+  w->writeC(0);
+  FEATURE_END;
+}
 void DivInstrument::putInsData2(SafeWriter* w, bool fui, const DivSong* song, bool insName) {
   size_t blockStartSeek=0;
   size_t blockEndSeek=0;
@@ -1178,6 +1270,7 @@ void DivInstrument::putInsData2(SafeWriter* w, bool fui, const DivSong* song, bo
   bool featurePN=false;
   bool featureS2=false;
   bool featureS3=false;
+  bool featureXA=false;
 
   bool checkForWL=false;
 
@@ -1575,6 +1668,9 @@ void DivInstrument::putInsData2(SafeWriter* w, bool fui, const DivSong* song, bo
     }
   }
 
+  if (!xattrs.empty()) {
+    featureXA = true;
+  }
   // write features
   if (featureNA) {
     writeFeatureNA(w);
@@ -1647,6 +1743,9 @@ void DivInstrument::putInsData2(SafeWriter* w, bool fui, const DivSong* song, bo
   if (featureS3) {
     writeFeatureS3(w);
   }
+  if (featureXA) {
+    writeFeatureXA(w);
+  }
 
   if (fui && (featureSL || featureWL)) {
     w->write("EN",2);
@@ -2581,6 +2680,71 @@ void DivInstrument::readFeatureS3(SafeReader& reader, short version) {
   READ_FEAT_END;
 }
 
+void DivInstrument::readFeatureXA(SafeReader& reader, short version) {
+  READ_FEAT_BEGIN;
+  DivInstrumentXattr xattr;
+
+  while (true) {
+    xattr.name=reader.readString();
+
+    if (!xattr.name.length()) {
+      break;
+    }
+    xattr.type=(DivXattrType)reader.readC();
+    if ((xattr.type & ~1) == DIV_XATTR_BOOLEAN) {
+      xattr.bool_val = xattr.type & 1;
+      xattr.type = DIV_XATTR_BOOLEAN;
+      xattrs.push_back(xattr);
+      xattr = DivInstrumentXattr();
+      continue;
+    }
+
+    switch (xattr.type) {
+      case DIV_XATTR_STRING:
+      xattr.str_val=reader.readString();
+      break;
+      case DIV_XATTR_UINT: {
+        unsigned char byte=0;
+        unsigned int shift_count=0;
+        do {
+          byte = reader.readC();
+          xattr.uint_val |= (byte & 0x7F) << shift_count;
+          shift_count += 7;
+        } while (byte & 0x80);
+        break;
+      }
+      case DIV_XATTR_INT: {
+        unsigned char byte=reader.readC();
+        unsigned char sign_bit=0;
+        unsigned int shift_count=6;
+
+        sign_bit = byte & 0x40;
+        xattr.uint_val |= byte & 0x3F;
+        while (byte & 0x80) {
+          byte=reader.readC();
+          xattr.uint_val |= (byte & 0x7F) << shift_count;
+          shift_count += 7;
+        }
+
+        // if the sign bit is on in the first byte of the VLQ encoded integer,
+        // negate the integer
+        if (sign_bit) {
+          xattr.int_val = -xattr.int_val;
+        }
+        break;
+      }
+      case DIV_XATTR_FLOAT32:
+      xattr.float_val = reader.readF();
+      break;
+      default:
+      // this should be unreachable
+      break;
+    }
+    xattrs.push_back(xattr);
+    xattr = DivInstrumentXattr();
+  }
+  READ_FEAT_END;
+}
 DivDataErrors DivInstrument::readInsDataNew(SafeReader& reader, short version, bool fui, DivSong* song) {
   unsigned char featCode[2];
   bool volIsCutoff=false;
@@ -2657,6 +2821,8 @@ DivDataErrors DivInstrument::readInsDataNew(SafeReader& reader, short version, b
       readFeatureS2(reader,version);
     } else if (memcmp(featCode,"S3",2)==0) { // SID3
       readFeatureS3(reader,version);
+    } else if (memcmp(featCode,"XA",2)==0) { // extended attributes
+      readFeatureXA(reader,version);
     } else {
       if (song==NULL && (memcmp(featCode,"SL",2)==0 || (memcmp(featCode,"WL",2)==0))) {
         // nothing
@@ -3749,5 +3915,6 @@ DivInstrument& DivInstrument::operator=( const DivInstrument& ins ) {
   // undo/redo history is specifically not copied
   *(DivInstrumentPOD*)this=ins;
   name=ins.name;
+  xattrs=ins.xattrs;
   return *this;
 }

src/engine/instrument.h

@@ -24,6 +24,8 @@
 #include "../ta-utils.h"
 #include "../pch.h"
 #include "../fixedQueue.h"
+#include "../hashUtils.h"
+#include <vector>
 
 struct DivSong;
 struct DivInstrument;
@@ -149,6 +151,17 @@ enum DivMacroTypeOp: unsigned char {
   DIV_MACRO_OP_KSR,
 };
 
+enum DivXattrType: unsigned char {
+  DIV_XATTR_STRING,
+  DIV_XATTR_UINT,
+  DIV_XATTR_INT,
+  DIV_XATTR_FLOAT32,
+
+  // the bool value is stored directly in the type
+  // as the least significant bit.
+  DIV_XATTR_BOOLEAN,
+};
+
 // FM operator structure:
 // - OPN:
 //   - AM, AR, DR, MULT, RR, SL, TL, RS, DT, D2R, SSG-EG
@@ -289,6 +302,55 @@ struct DivInstrumentMacro {
   }
 };
 
+struct DivInstrumentXattr {
+  String name;
+  DivXattrType type;
+
+  String str_val;
+  union {
+    unsigned int uint_val;
+    int int_val;
+    float float_val;
+    bool bool_val;
+  };
+
+  DivInstrumentXattr():
+    name("example.empty"),
+    type(DIV_XATTR_STRING),
+    str_val(),
+    int_val(0) {};
+};
+
+// Hasher object for Xattr
+template<>
+struct std::hash<DivInstrumentXattr>
+{
+  size_t operator()(const DivInstrumentXattr& xattr) const noexcept {
+    size_t hash=0;
+    hash=combineHash(hash, xattr.name);
+    hash=combineHash(hash, xattr.type);
+
+    switch (xattr.type) {
+      case DIV_XATTR_STRING:
+      hash=combineHash(hash, xattr.str_val);
+      break;
+      case DIV_XATTR_UINT:
+      hash=combineHash(hash, xattr.uint_val);
+      break;
+      case DIV_XATTR_INT:
+      hash=combineHash(hash, xattr.int_val);
+      break;
+      case DIV_XATTR_FLOAT32:
+      hash=combineHash(hash, xattr.float_val);
+      break;
+      case DIV_XATTR_BOOLEAN:
+      hash=combineHash(hash, xattr.bool_val);
+      break;
+    }
+    return hash;
+  }
+};
+
 struct DivInstrumentSTD {
   DivInstrumentMacro volMacro;
   DivInstrumentMacro arpMacro;
@@ -1029,12 +1091,16 @@ struct DivInstrumentUndoStep {
   DivInstrumentUndoStep() :
     name(""),
     nameValid(false),
+    xattrsValid(false),
     processTime(0) {
   }
 
   MemPatch podPatch;
   String name;
   bool nameValid;
+
+  std::vector<DivInstrumentXattr> xattrs;
+  bool xattrsValid;
   size_t processTime;
 
   void applyAndReverse(DivInstrument* target);
@@ -1043,6 +1109,7 @@ struct DivInstrumentUndoStep {
 
 struct DivInstrument : DivInstrumentPOD {
   String name;
+  std::vector<DivInstrumentXattr> xattrs;
 
   DivInstrument() :
     name("") {
@@ -1095,6 +1162,7 @@ struct DivInstrument : DivInstrumentPOD {
   void writeFeaturePN(SafeWriter* w);
   void writeFeatureS2(SafeWriter* w);
   void writeFeatureS3(SafeWriter* w);
+  void writeFeatureXA(SafeWriter* w);
 
   void readFeatureNA(SafeReader& reader, short version);
   void readFeatureFM(SafeReader& reader, short version);
@@ -1119,6 +1187,7 @@ struct DivInstrument : DivInstrumentPOD {
   void readFeaturePN(SafeReader& reader, short version);
   void readFeatureS2(SafeReader& reader, short version);
   void readFeatureS3(SafeReader& reader, short version);
+  void readFeatureXA(SafeReader& reader, short version);
 
   DivDataErrors readInsDataOld(SafeReader& reader, short version);
   DivDataErrors readInsDataNew(SafeReader& reader, short version, bool fui, DivSong* song);