1 package edu.jiangxin.apktoolbox.convert.protobuf.unsupervised;
2
3 import com.google.protobuf.WireFormat;
4 import org.apache.commons.lang3.ArrayUtils;
5 import org.json.JSONObject;
6
7 import java.math.BigInteger;
8 import java.util.ArrayList;
9 import java.util.HashMap;
10 import java.util.List;
11 import java.util.Map;
12
13
14 public class ProtobufDecoder {
15
16 private static final String VALID_DATA = "valid_data";
17
18 private static final String LEFT_OVER_DATA = "left_over_data";
19
20 private static final String KEY_BYTE_RANGE = "byteRange";
21
22 private static final String KEY_FIELD_NUMBER = "fieldNumber";
23
24 private static final String KEY_TYPE = "type";
25
26 private static final String KEY_VALUE = "value";
27
28
29
30
31
32
33
34 public static String bytesDecoder(byte[] bytes) {
35 Map<String, Object> map = decodeProto(bytes);
36 List<Map<String, Object>> validItems = (List<Map<String, Object>>) map.get(VALID_DATA);
37 List<JSONObject> result = new ArrayList<>();
38 for (Map<String, Object> validItem : validItems) {
39 DecoderResult protobufPart = getProtobufPart(validItem);
40 result.add(protobufPart.toJson());
41 }
42 JSONObject jsonObject = new JSONObject();
43 jsonObject.put(VALID_DATA, result);
44 jsonObject.put(LEFT_OVER_DATA, map.get(LEFT_OVER_DATA));
45 return jsonObject.toString(2);
46 }
47
48
49
50
51
52
53
54 private static Map<String, Object> decodeProto(byte[] buffer) {
55 BufferReader reader = new BufferReader(buffer);
56 List<Object> parts = new ArrayList<>();
57 reader.trySkipGrpcHeader();
58
59 try {
60 while (reader.leftBytes() > 0) {
61 reader.checkpoint();
62
63 List<Integer> byteRange = new ArrayList<>();
64 byteRange.add(reader.getOffset());
65 int indexType = reader.readVarInt().intValue();
66 int type = indexType & 0b111;
67 int fieldNumber = indexType >> 3;
68
69 Object value;
70 if (type == WireFormat.WIRETYPE_VARINT) {
71 value = reader.readVarInt();
72 } else if (type == WireFormat.WIRETYPE_LENGTH_DELIMITED) {
73 int length = reader.readVarInt().intValue();
74 value = reader.readBuffer(length);
75 } else if (type == WireFormat.WIRETYPE_FIXED32) {
76 value = reader.readBuffer(4);
77 } else if (type == WireFormat.WIRETYPE_FIXED64) {
78 value = reader.readBuffer(8);
79 } else {
80 throw new RuntimeException("Unknown type: " + type);
81 }
82 byteRange.add(reader.getOffset());
83
84 Map<String, Object> part = new HashMap<>();
85 part.put(KEY_BYTE_RANGE, byteRange);
86 part.put(KEY_FIELD_NUMBER, fieldNumber);
87 part.put(KEY_TYPE, type);
88 part.put(KEY_VALUE, value);
89 parts.add(part);
90 }
91 } catch (Exception e) {
92 reader.resetToCheckpoint();
93 }
94
95 Map<String, Object> result = new HashMap<>();
96 result.put(VALID_DATA, parts);
97 result.put(LEFT_OVER_DATA, ByteUtil.bytesToHex(reader.readBuffer(reader.leftBytes())));
98
99 return result;
100 }
101
102
103
104
105
106
107
108 private static DecoderResult getProtobufPart(Map<String, Object> part) {
109 DecoderResult result = new DecoderResult();
110 result.setByteRange((List<Integer>) part.get(KEY_BYTE_RANGE));
111 result.setFieldNumber((int) part.get(KEY_FIELD_NUMBER));
112 int type = (int) part.get(KEY_TYPE);
113 Object value = part.get(KEY_VALUE);
114 switch (type) {
115 case WireFormat.WIRETYPE_VARINT:
116 String valueStr = value.toString();
117 List<Map<String, Object>> varintResult = decodeVarintParts(valueStr);
118 result.setContent(JSONObject.valueToString(varintResult));
119 break;
120 case WireFormat.WIRETYPE_LENGTH_DELIMITED:
121 byte[] bytes = (byte[]) value;
122 Map<String, Object> decoded = decodeProto(bytes);
123 String leftOverData = (String) decoded.get(LEFT_OVER_DATA);
124 if (bytes != null && bytes.length > 0 && leftOverData != null && leftOverData.length() == 0) {
125 List<Map<String, Object>> decodedParts = (List<Map<String, Object>>) decoded.get(VALID_DATA);
126 List<DecoderResult> subResults = new ArrayList<>();
127 for (Map<String, Object> decodedPart : decodedParts) {
128 DecoderResult protobufPart = getProtobufPart(decodedPart);
129 subResults.add(protobufPart);
130 }
131 result.setSubResults(subResults);
132 } else {
133 Map<String, Object> map = decodeStringOrBytes(bytes);
134 result.setContent((String) map.get(KEY_VALUE));
135 }
136 break;
137 case WireFormat.WIRETYPE_FIXED64:
138 bytes = (byte[]) value;
139 List<Map<String, Object>> fixed64Result = decodeFixed64(bytes);
140 result.setContent(JSONObject.valueToString(fixed64Result));
141 break;
142 case WireFormat.WIRETYPE_FIXED32:
143 bytes = (byte[]) value;
144 List<Map<String, Object>> fixed32Result = decodeFixed32(bytes);
145 result.setContent(JSONObject.valueToString(fixed32Result));
146 break;
147 default:
148 break;
149 }
150 result.setType(type);
151 return result;
152 }
153
154
155
156
157
158
159 private static List<Map<String, Object>> decodeFixed32(byte[] value) {
160 float floatValue = ByteUtil.bytesToFloat(value);
161 int intValue = ByteUtil.bytesToInt(value);
162 int uintValue = ByteUtil.bytesToInt(value);
163
164 List<Map<String, Object>> result = new ArrayList<>(3);
165 Map<String, Object> map1 = new HashMap<>(2);
166 map1.put(KEY_TYPE, "Int");
167 map1.put(KEY_VALUE, intValue);
168 result.add(map1);
169
170 if (intValue != uintValue) {
171 Map<String, Object> map2 = new HashMap<>(2);
172 map2.put(KEY_TYPE, "Unsigned Int");
173 map2.put(KEY_VALUE, uintValue);
174 result.add(map2);
175 }
176 Map<String, Object> map3 = new HashMap<>(2);
177 map3.put(KEY_TYPE, "Float");
178 map3.put(KEY_VALUE, floatValue);
179 result.add(map3);
180 return result;
181 }
182
183
184
185
186
187
188 private static List<Map<String, Object>> decodeFixed64(byte[] value) {
189 double floatValue = ByteUtil.bytesToDouble(value);
190 BigInteger intValue = new BigInteger(ByteUtil.bytesToHex(value), 16);
191 BigInteger uintValue = twoComplements(intValue);
192
193 List<Map<String, Object>> result = new ArrayList<>(3);
194 Map<String, Object> map1 = new HashMap<>(2);
195 map1.put(KEY_TYPE, "Int");
196 map1.put(KEY_VALUE, intValue.toString());
197 result.add(map1);
198
199 if (!intValue.equals(uintValue)) {
200 Map<String, Object> map2 = new HashMap<>(2);
201 map2.put(KEY_TYPE, "Unsigned Int");
202 map2.put(KEY_VALUE, uintValue.toString());
203 result.add(map2);
204 }
205 Map<String, Object> map3 = new HashMap<>(2);
206 map3.put(KEY_TYPE, "Double");
207 map3.put(KEY_VALUE, floatValue);
208 result.add(map3);
209 return result;
210 }
211
212
213
214
215
216
217 private static Map<String, Object> decodeStringOrBytes(byte[] value) {
218 Map<String, Object> result = new HashMap<>(2);
219 if (ArrayUtils.isEmpty(value)) {
220 result.put(KEY_TYPE, "string|bytes");
221 result.put(KEY_VALUE, "");
222 return result;
223 }
224 try {
225 result.put(KEY_TYPE, "string");
226 result.put(KEY_VALUE, hexStrToStr(ByteUtil.bytesToHex(value), "utf-8"));
227 } catch (Exception e) {
228 result.put(KEY_TYPE, "bytes");
229 result.put(KEY_VALUE, ByteUtil.bytesToHex(value));
230 }
231 return result;
232 }
233
234
235
236
237
238
239 private static List<Map<String, Object>> decodeVarintParts(String value) {
240 List<Map<String, Object>> result = new ArrayList<>(3);
241 BigInteger intVal = new BigInteger(value);
242 Map<String, Object> map1 = new HashMap<>(2);
243 map1.put(KEY_TYPE, "Int");
244 map1.put(KEY_VALUE, String.valueOf(intVal));
245 result.add(map1);
246
247 BigInteger signedIntVal = VarintUtils.interpretAsSignedType(intVal);
248 if (!signedIntVal.equals(intVal)) {
249 Map<String, Object> map2 = new HashMap<>(2);
250 map2.put(KEY_TYPE, "Signed Int");
251 map2.put(KEY_VALUE, String.valueOf(signedIntVal));
252 result.add(map2);
253 }
254 return result;
255 }
256
257
258
259
260
261
262
263
264 private static BigInteger twoComplements(BigInteger uintValue) {
265 if (uintValue.compareTo(new BigInteger("7fffffffffffffff", 16)) > 0) {
266 return uintValue.subtract(new BigInteger("10000000000000000", 16));
267 } else {
268 return uintValue;
269 }
270 }
271
272
273
274
275
276
277
278
279 private static String hexStrToStr(String string, String charsetName) {
280 if (string == null || string.equals("")) {
281 return null;
282 }
283 byte[] baKeyword = new byte[string.length() / 2];
284 for (int i = 0; i < baKeyword.length; i++) {
285 try {
286 baKeyword[i] = (byte) (0xff & Integer.parseInt(string.substring(i * 2, i * 2 + 2), 16));
287 } catch (Exception e) {
288 e.printStackTrace();
289 }
290 }
291 try {
292 string = new String(baKeyword, charsetName);
293 } catch (Exception e1) {
294 e1.printStackTrace();
295 }
296 return string;
297 }
298 }