UTMLTokenizerDataAsset

: UPrimaryDataAsset : UDataAsset : UObject

/Script/TriiodideML.TMLTokenizerDataAsset 0x00B8 (184 B) align 8 inherited 0x0030 vtable /Script/TriiodideML

0x00x00B8

Members 9

Show inherited members

Size	Name	Type	Flags	Default
0x10	Vocabulary	TArray<FString>	6 flags	[]
0x04	EmbeddingDimension	int32	8 flags	1024
0x04	MaxSequenceLength	int32	8 flags	512
0x04	PadTokenId	int32	8 flags	0
0x04	EosTokenId	int32	8 flags	1
0x04	BosTokenId	int32	8 flags	2
0x04	UnkTokenId	int32	8 flags	3
0x10	SourceTokenizerFile	FFilePath	2 flags	{FilePath=""}
0x50	TokenToId	TMap<FString, int32>	Transient+1 flags	<TMap>

Inherited from UDataAsset 1

Offset	Size	Name	Type	Flags	Default
	0x08	NativeClass	TSubclassOf<UDataAsset>	6 flags	/Script/Engine.DataAsset

Functions 5

Signature	Flags	Params
bool IsEncodeLookupBuilt()	FinalNative+1 flags	0x01
int32 GetVocabSize()	FinalNative+1 flags	0x04
int32 GetTokenId(const FString& Token)	FinalNative+1 flags	0x14
FString GetToken(int32 TokenId)	FinalNative+1 flags	0x18
void BuildEncodeLookup()	FinalNative+1 flags	0x00

// /Script/TriiodideML.TMLTokenizerDataAsset
// size 0x00B8 (184), inherited 0x0030, align 8
class UTMLTokenizerDataAsset : public UPrimaryDataAsset
{
public:
    TArray<FString> Vocabulary;                        // 0x0030(0x10) Edit BlueprintVisible BlueprintReadOnly ZeroConstructor EditConst NativeAccessSpecifierPublic = []
    int32 EmbeddingDimension;                          // 0x0040(0x04) Edit BlueprintVisible BlueprintReadOnly ZeroConstructor IsPlainOldData NoDestructor HasGetValueTypeHash NativeAccessSpecifierPublic = 1024
    int32 MaxSequenceLength;                           // 0x0044(0x04) Edit BlueprintVisible BlueprintReadOnly ZeroConstructor IsPlainOldData NoDestructor HasGetValueTypeHash NativeAccessSpecifierPublic = 512
    int32 PadTokenId;                                  // 0x0048(0x04) Edit BlueprintVisible BlueprintReadOnly ZeroConstructor IsPlainOldData NoDestructor HasGetValueTypeHash NativeAccessSpecifierPublic = 0
    int32 EosTokenId;                                  // 0x004C(0x04) Edit BlueprintVisible BlueprintReadOnly ZeroConstructor IsPlainOldData NoDestructor HasGetValueTypeHash NativeAccessSpecifierPublic = 1
    int32 BosTokenId;                                  // 0x0050(0x04) Edit BlueprintVisible BlueprintReadOnly ZeroConstructor IsPlainOldData NoDestructor HasGetValueTypeHash NativeAccessSpecifierPublic = 2
    int32 UnkTokenId;                                  // 0x0054(0x04) Edit BlueprintVisible BlueprintReadOnly ZeroConstructor IsPlainOldData NoDestructor HasGetValueTypeHash NativeAccessSpecifierPublic = 3
    FFilePath SourceTokenizerFile;                     // 0x0058(0x10) Edit NativeAccessSpecifierPublic = {FilePath=""}
    TMap<FString, int32> TokenToId;                    // 0x0068(0x50) Transient NativeAccessSpecifierPrivate = <TMap>

    bool IsEncodeLookupBuilt(); // 0x757e1d0 [Final Native Public]
    int32 GetVocabSize(); // 0x3cff1d0 [Final Native Public]
    int32 GetTokenId(const FString& Token); // 0x757dbd0 [Final Native Public]
    FString GetToken(int32 TokenId); // 0x757db30 [Final Native Public]
    void BuildEncodeLookup(); // 0x757d960 [Final Native Public]
};