#How to deserialize []byte ?

5 messages · Page 1 of 1 (latest)

pastel junco
#

so i want to extract .safetensors to golang

i'm using bufio scanner, and not sure what to do next ?

modelSafetensors, err = os.Open(path)
    if err != nil {
        return nil, err
    }

    scanner := bufio.NewScanner(modelSafetensors)
    for scanner.Scan() {
        line = scanner.Text()
        rawb = scanner.Bytes()
        fmt.Printf("%#v\n", line)
        // fmt.Printf("%#v\n", rawb)
        fmt.Printf("\n\n")

    }
    if err := scanner.Err(); err != nil {
        fmt.Fprintln(os.Stderr, "reading standard input:", err)
    }

"0\xbf|\xb2\xbc\xbc\x84~\xe4\xbe\xfa\xa4#?eܰ\xbf$\xec{\xbf`㵾\xdbPʾ\x10\xc8ҾG\xe9S\xbfz\xd4\x14?\xb3/\x12\xbf\x01\xd4$\xbdY\x8c\x1a\xbf\xca\xeb\x8e>\x81Ε\xbe\x97\xb2!>\x9b\x96\x9b>\xe0 \x98>)\x02x?\x94̧?\xf2)\x10?E|\x13\xbe\xc90\x80\xbf\x96\xed\xbf\xbf\x97\x98\x14\xc0\xab:V\xbeD\\v@=\x9e\xcb?\xf3\xbc\f\xbe\x17\x02\xaf\xbe\I\x82?M슾Ř\t>&\xa0/\xbf\xee\u0089>ը\xe4?,\x8f\xb4?\xfbc#\xbfK\xc9?\xbeM>'?OO\x90>O\xc0g?\x84\xcf\b\xbe)=,?uE\x87\xbf\x06\xe4*\xbf\xebu\x05?t\xae\x8c\xbf\x1f\u0088\xbf\xc3\xce\xef?\xa4\xfc\xab\xbe\xc7\xf0\x81\xbf\xe5\xe4\r\xc0rD9\xbf\xff\xb8*>\xf2o\xa8?\xfaF\xe2\xbf;\xc7S?C{|\xbf\x91K\xf5\xbf\xfa4q\xbe\xeaΑ\xbd\t,پ\xd7h\xf6\xbe0K!@'\x8e\x8d?.G\xf3\xbdO\x99a>\x91'\xc6=\x12\x97\xc0\xbeJ"\xb2>\xe8\x93Ŀ\xf9\xc7\xe0>\xe8\x9e轼\xc9"\xbe\xb7>9\xbe\xed\xdf\xc0\xbfw\xe1\x1c\xbf)q\xfa\xbfM\xb9"

#

How to deserialize []byte ?

#

while i don't know the exact endianness, i have the test cases

def test_serialization(self):
        data = np.zeros((2, 2), dtype=np.int32)
        out = save({"test": data})

        self.assertEqual(
            out,
            b'@\x00\x00\x00\x00\x00\x00\x00{"test":{"dtype":"I32","shape":[2,2],"data_offsets":[0,16]}}   '
            b" \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00",
        )

        save_file({"test": data}, "serialization.safetensors")
        out = open("serialization.safetensors", "rb").read()
        self.assertEqual(
            out,
            b'@\x00\x00\x00\x00\x00\x00\x00{"test":{"dtype":"I32","shape":[2,2],"data_offsets":[0,16]}}   '
            b" \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00",
        )

        data[1, 1] = 1
        out = save({"test": data})

        self.assertEqual(
            out,
            b'@\x00\x00\x00\x00\x00\x00\x00{"test":{"dtype":"I32","shape":[2,2],"data_offsets":[0,16]}}   '
            b" \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00",
        )
        save_file({"test": data}, "serialization.safetensors")
        out = open("serialization.safetensors", "rb").read()
        self.assertEqual(
            out,
            b'@\x00\x00\x00\x00\x00\x00\x00{"test":{"dtype":"I32","shape":[2,2],"data_offsets":[0,16]}}   '
            b" \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00",
        )

    def test_deserialization(self):
        serialized = b"""<\x00\x00\x00\x00\x00\x00\x00{"test":{"dtype":"I32","shape":[2,2],"data_offsets":[0,16]}}\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"""

        out = load(serialized)
        self.assertEqual(list(out.keys()), ["test"])
        np.testing.assert_array_equal(out["test"], np.zeros((2, 2), dtype=np.int32))
# 

    def test_deserialization_metadata(self):
        serialized = (
            b'f\x00\x00\x00\x00\x00\x00\x00{"__metadata__":{"framework":"pt"},"test1":{"dtype":"I32","shape":[2,2],"data_offsets":[0,16]}}'
            b"       \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
        )

        with tempfile.NamedTemporaryFile() as f:
            f.write(serialized)
            f.seek(0)

            with safe_open(f.name, framework="np") as g:
                self.assertEqual(g.metadata(), {"framework": "pt"})

    def test_serialization_metadata(self):
        data = np.zeros((2, 2), dtype=np.int32)
        out1 = save({"test1": data}, metadata={"framework": "pt"})
        self.assertEqual(
            out1,
            b'`\x00\x00\x00\x00\x00\x00\x00{"__metadata__":{"framework":"pt"},"test1":{"dtype":"I32","shape":[2,2],"data_offsets":[0,16]}}'
            b" \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00",
        )
        self.assertEqual(out1[8:].index(b"\x00") + 8, 104)
        self.assertEqual((out1[8:].index(b"\x00") + 8) % 8, 0)

    def test_serialization_no_big_endian(self):
        # Big endian tensor
        data = np.zeros((2, 2), dtype=">i4")
        out1 = save({"test1": data}, metadata={"framework": "pt"})
        self.assertEqual(
            out1,
            b'`\x00\x00\x00\x00\x00\x00\x00{"__metadata__":{"framework":"pt"},"test1":{"dtype":"I32","shape":[2,2],"data_offsets":[0,16]}}'
            b" \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00",
        )
        self.assertEqual(out1[8:].index(b"\x00") + 8, 104)
        self.assertEqual((out1[8:].index(b"\x00") + 8) % 8, 0)
modern bear