From e3cafeaf9292f67459ff1d186f68283bfaedf2ae Mon Sep 17 00:00:00 2001 From: Wim Date: Mon, 31 Jan 2022 00:27:37 +0100 Subject: Add dependencies/vendor (whatsapp) --- vendor/go.mau.fi/whatsmeow/appstate/decode.go | 310 +++++++++++++++++++++ vendor/go.mau.fi/whatsmeow/appstate/errors.go | 19 ++ vendor/go.mau.fi/whatsmeow/appstate/hash.go | 96 +++++++ vendor/go.mau.fi/whatsmeow/appstate/keys.go | 85 ++++++ .../go.mau.fi/whatsmeow/appstate/lthash/lthash.go | 58 ++++ 5 files changed, 568 insertions(+) create mode 100644 vendor/go.mau.fi/whatsmeow/appstate/decode.go create mode 100644 vendor/go.mau.fi/whatsmeow/appstate/errors.go create mode 100644 vendor/go.mau.fi/whatsmeow/appstate/hash.go create mode 100644 vendor/go.mau.fi/whatsmeow/appstate/keys.go create mode 100644 vendor/go.mau.fi/whatsmeow/appstate/lthash/lthash.go (limited to 'vendor/go.mau.fi/whatsmeow/appstate') diff --git a/vendor/go.mau.fi/whatsmeow/appstate/decode.go b/vendor/go.mau.fi/whatsmeow/appstate/decode.go new file mode 100644 index 00000000..5c895470 --- /dev/null +++ b/vendor/go.mau.fi/whatsmeow/appstate/decode.go @@ -0,0 +1,310 @@ +// Copyright (c) 2021 Tulir Asokan +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package appstate + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "fmt" + + "google.golang.org/protobuf/proto" + + waBinary "go.mau.fi/whatsmeow/binary" + waProto "go.mau.fi/whatsmeow/binary/proto" + "go.mau.fi/whatsmeow/store" + "go.mau.fi/whatsmeow/util/cbcutil" +) + +// PatchList represents a decoded response to getting app state patches from the WhatsApp servers. +type PatchList struct { + Name WAPatchName + HasMorePatches bool + Patches []*waProto.SyncdPatch + Snapshot *waProto.SyncdSnapshot +} + +// DownloadExternalFunc is a function that can download a blob of external app state patches. +type DownloadExternalFunc func(*waProto.ExternalBlobReference) ([]byte, error) + +func parseSnapshotInternal(collection *waBinary.Node, downloadExternal DownloadExternalFunc) (*waProto.SyncdSnapshot, error) { + snapshotNode := collection.GetChildByTag("snapshot") + rawSnapshot, ok := snapshotNode.Content.([]byte) + if snapshotNode.Tag != "snapshot" || !ok { + return nil, nil + } + var snapshot waProto.ExternalBlobReference + err := proto.Unmarshal(rawSnapshot, &snapshot) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal snapshot: %w", err) + } + var rawData []byte + rawData, err = downloadExternal(&snapshot) + if err != nil { + return nil, fmt.Errorf("failed to download external mutations: %w", err) + } + var downloaded waProto.SyncdSnapshot + err = proto.Unmarshal(rawData, &downloaded) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal mutation list: %w", err) + } + return &downloaded, nil +} + +func parsePatchListInternal(collection *waBinary.Node, downloadExternal DownloadExternalFunc) ([]*waProto.SyncdPatch, error) { + patchesNode := collection.GetChildByTag("patches") + patchNodes := patchesNode.GetChildren() + patches := make([]*waProto.SyncdPatch, 0, len(patchNodes)) + for i, patchNode := range patchNodes { + rawPatch, ok := patchNode.Content.([]byte) + if patchNode.Tag != "patch" || !ok { + continue + } + var patch waProto.SyncdPatch + err := proto.Unmarshal(rawPatch, &patch) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal patch #%d: %w", i+1, err) + } + if patch.GetExternalMutations() != nil && downloadExternal != nil { + var rawData []byte + rawData, err = downloadExternal(patch.GetExternalMutations()) + if err != nil { + return nil, fmt.Errorf("failed to download external mutations: %w", err) + } + var downloaded waProto.SyncdMutations + err = proto.Unmarshal(rawData, &downloaded) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal mutation list: %w", err) + } else if len(downloaded.GetMutations()) == 0 { + return nil, fmt.Errorf("didn't get any mutations from download") + } + patch.Mutations = downloaded.Mutations + } + patches = append(patches, &patch) + } + return patches, nil +} + +// ParsePatchList will decode an XML node containing app state patches, including downloading any external blobs. +func ParsePatchList(node *waBinary.Node, downloadExternal DownloadExternalFunc) (*PatchList, error) { + collection := node.GetChildByTag("sync", "collection") + ag := collection.AttrGetter() + snapshot, err := parseSnapshotInternal(&collection, downloadExternal) + if err != nil { + return nil, err + } + patches, err := parsePatchListInternal(&collection, downloadExternal) + if err != nil { + return nil, err + } + list := &PatchList{ + Name: WAPatchName(ag.String("name")), + HasMorePatches: ag.OptionalBool("has_more_patches"), + Patches: patches, + Snapshot: snapshot, + } + return list, ag.Error() +} + +type patchOutput struct { + RemovedMACs [][]byte + AddedMACs []store.AppStateMutationMAC + Mutations []Mutation +} + +func (proc *Processor) decodeMutations(mutations []*waProto.SyncdMutation, out *patchOutput, validateMACs bool) error { + for i, mutation := range mutations { + keyID := mutation.GetRecord().GetKeyId().GetId() + keys, err := proc.getAppStateKey(keyID) + if err != nil { + return fmt.Errorf("failed to get key %X to decode mutation: %w", keyID, err) + } + content := mutation.GetRecord().GetValue().GetBlob() + content, valueMAC := content[:len(content)-32], content[len(content)-32:] + if validateMACs { + expectedValueMAC := generateContentMAC(mutation.GetOperation(), content, keyID, keys.ValueMAC) + if !bytes.Equal(expectedValueMAC, valueMAC) { + return fmt.Errorf("failed to verify mutation #%d: %w", i+1, ErrMismatchingContentMAC) + } + } + iv, content := content[:16], content[16:] + plaintext, err := cbcutil.Decrypt(keys.ValueEncryption, iv, content) + if err != nil { + return fmt.Errorf("failed to decrypt mutation #%d: %w", i+1, err) + } + var syncAction waProto.SyncActionData + err = proto.Unmarshal(plaintext, &syncAction) + if err != nil { + return fmt.Errorf("failed to unmarshal mutation #%d: %w", i+1, err) + } + indexMAC := mutation.GetRecord().GetIndex().GetBlob() + if validateMACs { + expectedIndexMAC := concatAndHMAC(sha256.New, keys.Index, syncAction.Index) + if !bytes.Equal(expectedIndexMAC, indexMAC) { + return fmt.Errorf("failed to verify mutation #%d: %w", i+1, ErrMismatchingIndexMAC) + } + } + var index []string + err = json.Unmarshal(syncAction.GetIndex(), &index) + if err != nil { + return fmt.Errorf("failed to unmarshal index of mutation #%d: %w", i+1, err) + } + if mutation.GetOperation() == waProto.SyncdMutation_REMOVE { + out.RemovedMACs = append(out.RemovedMACs, indexMAC) + } else if mutation.GetOperation() == waProto.SyncdMutation_SET { + out.AddedMACs = append(out.AddedMACs, store.AppStateMutationMAC{ + IndexMAC: indexMAC, + ValueMAC: valueMAC, + }) + } + out.Mutations = append(out.Mutations, Mutation{ + Operation: mutation.GetOperation(), + Action: syncAction.GetValue(), + Index: index, + IndexMAC: indexMAC, + ValueMAC: valueMAC, + }) + } + return nil +} + +func (proc *Processor) storeMACs(name WAPatchName, currentState HashState, out *patchOutput) { + err := proc.Store.AppState.PutAppStateVersion(string(name), currentState.Version, currentState.Hash) + if err != nil { + proc.Log.Errorf("Failed to update app state version in the database: %v", err) + } + err = proc.Store.AppState.DeleteAppStateMutationMACs(string(name), out.RemovedMACs) + if err != nil { + proc.Log.Errorf("Failed to remove deleted mutation MACs from the database: %v", err) + } + err = proc.Store.AppState.PutAppStateMutationMACs(string(name), currentState.Version, out.AddedMACs) + if err != nil { + proc.Log.Errorf("Failed to insert added mutation MACs to the database: %v", err) + } +} + +func (proc *Processor) validateSnapshotMAC(name WAPatchName, currentState HashState, keyID, expectedSnapshotMAC []byte) (keys ExpandedAppStateKeys, err error) { + keys, err = proc.getAppStateKey(keyID) + if err != nil { + err = fmt.Errorf("failed to get key %X to verify patch v%d MACs: %w", keyID, currentState.Version, err) + return + } + snapshotMAC := currentState.generateSnapshotMAC(name, keys.SnapshotMAC) + if !bytes.Equal(snapshotMAC, expectedSnapshotMAC) { + err = fmt.Errorf("failed to verify patch v%d: %w", currentState.Version, ErrMismatchingLTHash) + } + return +} + +func (proc *Processor) decodeSnapshot(name WAPatchName, ss *waProto.SyncdSnapshot, initialState HashState, validateMACs bool, newMutationsInput []Mutation) (newMutations []Mutation, currentState HashState, err error) { + currentState = initialState + currentState.Version = ss.GetVersion().GetVersion() + + encryptedMutations := make([]*waProto.SyncdMutation, len(ss.GetRecords())) + for i, record := range ss.GetRecords() { + encryptedMutations[i] = &waProto.SyncdMutation{ + Operation: waProto.SyncdMutation_SET.Enum(), + Record: record, + } + } + + var warn []error + warn, err = currentState.updateHash(encryptedMutations, func(indexMAC []byte, maxIndex int) ([]byte, error) { + return nil, nil + }) + if len(warn) > 0 { + proc.Log.Warnf("Warnings while updating hash for %s: %+v", name, warn) + } + if err != nil { + err = fmt.Errorf("failed to update state hash: %w", err) + return + } + + if validateMACs { + _, err = proc.validateSnapshotMAC(name, currentState, ss.GetKeyId().GetId(), ss.GetMac()) + if err != nil { + return + } + } + + var out patchOutput + out.Mutations = newMutationsInput + err = proc.decodeMutations(encryptedMutations, &out, validateMACs) + if err != nil { + err = fmt.Errorf("failed to decode snapshot of v%d: %w", currentState.Version, err) + return + } + proc.storeMACs(name, currentState, &out) + newMutations = out.Mutations + return +} + +// DecodePatches will decode all the patches in a PatchList into a list of app state mutations. +func (proc *Processor) DecodePatches(list *PatchList, initialState HashState, validateMACs bool) (newMutations []Mutation, currentState HashState, err error) { + currentState = initialState + var expectedLength int + if list.Snapshot != nil { + expectedLength = len(list.Snapshot.GetRecords()) + } + for _, patch := range list.Patches { + expectedLength += len(patch.GetMutations()) + } + newMutations = make([]Mutation, 0, expectedLength) + + if list.Snapshot != nil { + newMutations, currentState, err = proc.decodeSnapshot(list.Name, list.Snapshot, currentState, validateMACs, newMutations) + if err != nil { + return + } + } + + for _, patch := range list.Patches { + version := patch.GetVersion().GetVersion() + currentState.Version = version + var warn []error + warn, err = currentState.updateHash(patch.GetMutations(), func(indexMAC []byte, maxIndex int) ([]byte, error) { + for i := maxIndex - 1; i >= 0; i-- { + if bytes.Equal(patch.Mutations[i].GetRecord().GetIndex().GetBlob(), indexMAC) { + value := patch.Mutations[i].GetRecord().GetValue().GetBlob() + return value[len(value)-32:], nil + } + } + // Previous value not found in current patch, look in the database + return proc.Store.AppState.GetAppStateMutationMAC(string(list.Name), indexMAC) + }) + if len(warn) > 0 { + proc.Log.Warnf("Warnings while updating hash for %s: %+v", list.Name, warn) + } + if err != nil { + err = fmt.Errorf("failed to update state hash: %w", err) + return + } + + if validateMACs { + var keys ExpandedAppStateKeys + keys, err = proc.validateSnapshotMAC(list.Name, currentState, patch.GetKeyId().GetId(), patch.GetSnapshotMac()) + if err != nil { + return + } + patchMAC := generatePatchMAC(patch, list.Name, keys.PatchMAC) + if !bytes.Equal(patchMAC, patch.GetPatchMac()) { + err = fmt.Errorf("failed to verify patch v%d: %w", version, ErrMismatchingPatchMAC) + return + } + } + + var out patchOutput + out.Mutations = newMutations + err = proc.decodeMutations(patch.GetMutations(), &out, validateMACs) + if err != nil { + return + } + proc.storeMACs(list.Name, currentState, &out) + newMutations = out.Mutations + } + return +} diff --git a/vendor/go.mau.fi/whatsmeow/appstate/errors.go b/vendor/go.mau.fi/whatsmeow/appstate/errors.go new file mode 100644 index 00000000..0668d86a --- /dev/null +++ b/vendor/go.mau.fi/whatsmeow/appstate/errors.go @@ -0,0 +1,19 @@ +// Copyright (c) 2021 Tulir Asokan +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package appstate + +import "errors" + +// Errors that this package can return. +var ( + ErrMissingPreviousSetValueOperation = errors.New("missing value MAC of previous SET operation") + ErrMismatchingLTHash = errors.New("mismatching LTHash") + ErrMismatchingPatchMAC = errors.New("mismatching patch MAC") + ErrMismatchingContentMAC = errors.New("mismatching content MAC") + ErrMismatchingIndexMAC = errors.New("mismatching index MAC") + ErrKeyNotFound = errors.New("didn't find app state key") +) diff --git a/vendor/go.mau.fi/whatsmeow/appstate/hash.go b/vendor/go.mau.fi/whatsmeow/appstate/hash.go new file mode 100644 index 00000000..4fdbabf6 --- /dev/null +++ b/vendor/go.mau.fi/whatsmeow/appstate/hash.go @@ -0,0 +1,96 @@ +// Copyright (c) 2021 Tulir Asokan +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package appstate + +import ( + "crypto/hmac" + "crypto/sha256" + "crypto/sha512" + "encoding/binary" + "fmt" + "hash" + + "go.mau.fi/whatsmeow/appstate/lthash" + waProto "go.mau.fi/whatsmeow/binary/proto" +) + +type Mutation struct { + Operation waProto.SyncdMutation_SyncdMutationSyncdOperation + Action *waProto.SyncActionValue + Index []string + IndexMAC []byte + ValueMAC []byte +} + +type HashState struct { + Version uint64 + Hash [128]byte +} + +func (hs *HashState) updateHash(mutations []*waProto.SyncdMutation, getPrevSetValueMAC func(indexMAC []byte, maxIndex int) ([]byte, error)) ([]error, error) { + var added, removed [][]byte + var warnings []error + + for i, mutation := range mutations { + if mutation.GetOperation() == waProto.SyncdMutation_SET { + value := mutation.GetRecord().GetValue().GetBlob() + added = append(added, value[len(value)-32:]) + } + indexMAC := mutation.GetRecord().GetIndex().GetBlob() + removal, err := getPrevSetValueMAC(indexMAC, i) + if err != nil { + return warnings, fmt.Errorf("failed to get value MAC of previous SET operation: %w", err) + } else if removal != nil { + removed = append(removed, removal) + } else if mutation.GetOperation() == waProto.SyncdMutation_REMOVE { + // TODO figure out if there are certain cases that are safe to ignore and others that aren't + // At least removing contact access from WhatsApp seems to create a REMOVE op for your own JID + // that points to a non-existent index and is safe to ignore here. Other keys might not be safe to ignore. + warnings = append(warnings, fmt.Errorf("%w for %X", ErrMissingPreviousSetValueOperation, indexMAC)) + //return ErrMissingPreviousSetValueOperation + } + } + + lthash.WAPatchIntegrity.SubtractThenAddInPlace(hs.Hash[:], removed, added) + return warnings, nil +} + +func uint64ToBytes(val uint64) []byte { + data := make([]byte, 8) + binary.BigEndian.PutUint64(data, val) + return data +} + +func concatAndHMAC(alg func() hash.Hash, key []byte, data ...[]byte) []byte { + h := hmac.New(alg, key) + for _, item := range data { + h.Write(item) + } + return h.Sum(nil) +} + +func (hs *HashState) generateSnapshotMAC(name WAPatchName, key []byte) []byte { + return concatAndHMAC(sha256.New, key, hs.Hash[:], uint64ToBytes(hs.Version), []byte(name)) +} + +func generatePatchMAC(patch *waProto.SyncdPatch, name WAPatchName, key []byte) []byte { + dataToHash := make([][]byte, len(patch.GetMutations())+3) + dataToHash[0] = patch.GetSnapshotMac() + for i, mutation := range patch.Mutations { + val := mutation.GetRecord().GetValue().GetBlob() + dataToHash[i+1] = val[len(val)-32:] + } + dataToHash[len(dataToHash)-2] = uint64ToBytes(patch.GetVersion().GetVersion()) + dataToHash[len(dataToHash)-1] = []byte(name) + return concatAndHMAC(sha256.New, key, dataToHash...) +} + +func generateContentMAC(operation waProto.SyncdMutation_SyncdMutationSyncdOperation, data, keyID, key []byte) []byte { + operationBytes := []byte{byte(operation) + 1} + keyDataLength := uint64ToBytes(uint64(len(keyID) + 1)) + return concatAndHMAC(sha512.New, key, operationBytes, keyID, data, keyDataLength)[:32] +} diff --git a/vendor/go.mau.fi/whatsmeow/appstate/keys.go b/vendor/go.mau.fi/whatsmeow/appstate/keys.go new file mode 100644 index 00000000..30ecbcc4 --- /dev/null +++ b/vendor/go.mau.fi/whatsmeow/appstate/keys.go @@ -0,0 +1,85 @@ +// Copyright (c) 2021 Tulir Asokan +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package appstate implements encoding and decoding WhatsApp's app state patches. +package appstate + +import ( + "encoding/base64" + "sync" + + "go.mau.fi/whatsmeow/store" + "go.mau.fi/whatsmeow/util/hkdfutil" + waLog "go.mau.fi/whatsmeow/util/log" +) + +// WAPatchName represents a type of app state patch. +type WAPatchName string + +const ( + // WAPatchCriticalBlock contains the user's settings like push name and locale. + WAPatchCriticalBlock WAPatchName = "critical_block" + // WAPatchCriticalUnblockLow contains the user's contact list. + WAPatchCriticalUnblockLow WAPatchName = "critical_unblock_low" + // WAPatchRegularLow contains some local chat settings like pin, archive status, and the setting of whether to unarchive chats when messages come in. + WAPatchRegularLow WAPatchName = "regular_low" + // WAPatchRegularHigh contains more local chat settings like mute status and starred messages. + WAPatchRegularHigh WAPatchName = "regular_high" + // WAPatchRegular contains protocol info about app state patches like key expiration. + WAPatchRegular WAPatchName = "regular" +) + +// AllPatchNames contains all currently known patch state names. +var AllPatchNames = [...]WAPatchName{WAPatchCriticalBlock, WAPatchCriticalUnblockLow, WAPatchRegularHigh, WAPatchRegular, WAPatchRegularLow} + +type Processor struct { + keyCache map[string]ExpandedAppStateKeys + keyCacheLock sync.Mutex + Store *store.Device + Log waLog.Logger +} + +func NewProcessor(store *store.Device, log waLog.Logger) *Processor { + return &Processor{ + keyCache: make(map[string]ExpandedAppStateKeys), + Store: store, + Log: log, + } +} + +type ExpandedAppStateKeys struct { + Index []byte + ValueEncryption []byte + ValueMAC []byte + SnapshotMAC []byte + PatchMAC []byte +} + +func expandAppStateKeys(keyData []byte) (keys ExpandedAppStateKeys) { + appStateKeyExpanded := hkdfutil.SHA256(keyData, nil, []byte("WhatsApp Mutation Keys"), 160) + return ExpandedAppStateKeys{appStateKeyExpanded[0:32], appStateKeyExpanded[32:64], appStateKeyExpanded[64:96], appStateKeyExpanded[96:128], appStateKeyExpanded[128:160]} +} + +func (proc *Processor) getAppStateKey(keyID []byte) (keys ExpandedAppStateKeys, err error) { + keyCacheID := base64.RawStdEncoding.EncodeToString(keyID) + var ok bool + + proc.keyCacheLock.Lock() + defer proc.keyCacheLock.Unlock() + + keys, ok = proc.keyCache[keyCacheID] + if !ok { + var keyData *store.AppStateSyncKey + keyData, err = proc.Store.AppStateKeys.GetAppStateSyncKey(keyID) + if keyData != nil { + keys = expandAppStateKeys(keyData.Data) + proc.keyCache[keyCacheID] = keys + } else if err == nil { + err = ErrKeyNotFound + } + } + return +} diff --git a/vendor/go.mau.fi/whatsmeow/appstate/lthash/lthash.go b/vendor/go.mau.fi/whatsmeow/appstate/lthash/lthash.go new file mode 100644 index 00000000..8d3045d6 --- /dev/null +++ b/vendor/go.mau.fi/whatsmeow/appstate/lthash/lthash.go @@ -0,0 +1,58 @@ +// Copyright (c) 2021 Tulir Asokan +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package lthash implements a summation based hash algorithm that maintains the +// integrity of a piece of data over a series of mutations. You can add/remove +// mutations, and it'll return a hash equal to if the same series of mutations +// was made sequentially. +package lthash + +import ( + "encoding/binary" + + "go.mau.fi/whatsmeow/util/hkdfutil" +) + +type LTHash struct { + HKDFInfo []byte + HKDFSize uint8 +} + +// WAPatchIntegrity is a LTHash instance initialized with the details used for verifying integrity of WhatsApp app state sync patches. +var WAPatchIntegrity = LTHash{[]byte("WhatsApp Patch Integrity"), 128} + +func (lth LTHash) SubtractThenAdd(base []byte, subtract, add [][]byte) []byte { + output := make([]byte, len(base)) + copy(output, base) + lth.SubtractThenAddInPlace(output, subtract, add) + return output +} + +func (lth LTHash) SubtractThenAddInPlace(base []byte, subtract, add [][]byte) { + lth.multipleOp(base, subtract, true) + lth.multipleOp(base, add, false) +} + +func (lth LTHash) multipleOp(base []byte, input [][]byte, subtract bool) { + for _, item := range input { + performPointwiseWithOverflow(base, hkdfutil.SHA256(item, nil, lth.HKDFInfo, lth.HKDFSize), subtract) + } +} + +func performPointwiseWithOverflow(base, input []byte, subtract bool) []byte { + for i := 0; i < len(base); i += 2 { + x := binary.LittleEndian.Uint16(base[i : i+2]) + y := binary.LittleEndian.Uint16(input[i : i+2]) + var result uint16 + if subtract { + result = x - y + } else { + result = x + y + } + binary.LittleEndian.PutUint16(base[i:i+2], result) + } + return base +} -- cgit v1.2.3