File tree Expand file tree Collapse file tree 1 file changed +0
-21
lines changed
test/Microsoft.ML.Tokenizers.Tests Expand file tree Collapse file tree 1 file changed +0
-21
lines changed Original file line number Diff line number Diff line change @@ -918,27 +918,6 @@ public void TestLargeInputConsistency(int length)
918918 var reconstructed = string . Concat ( tokens . Select ( t => t . Value ) ) ;
919919 Assert . Equal ( inputRepeated , reconstructed ) ;
920920 }
921-
922- [ Fact ]
923- public void TestLargeInputPerformance ( )
924- {
925- // Test that very large inputs complete in reasonable time
926- // This would timeout or take extremely long with O(n^2) algorithm
927-
928- string veryLargeInput = new string ( 'a' , 5000 ) ;
929- var stopwatch = System . Diagnostics . Stopwatch . StartNew ( ) ;
930- IReadOnlyList < int > ids = GPT4 . EncodeToIds ( veryLargeInput ) ;
931- stopwatch . Stop ( ) ;
932-
933- // Should complete in well under a second even for 5000 chars
934- // With O(n^2) this could take several seconds
935- Assert . True ( stopwatch . ElapsedMilliseconds < 5000 ,
936- $ "Large input encoding took { stopwatch . ElapsedMilliseconds } ms, expected < 5000ms") ;
937-
938- // Verify correctness
939- string decoded = GPT4 . Decode ( ids ) ;
940- Assert . Equal ( veryLargeInput , decoded ) ;
941- }
942921 }
943922}
944923
You can’t perform that action at this time.
0 commit comments