Skip to content

Autocast #1235

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 51 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
51 commits
Select commit Hold shift + click to select a range
51d1d95
Autocast
haytham2597 Feb 11, 2024
29b4900
Added some features
haytham2597 Feb 17, 2024
defd582
Fix mistake gitignore
haytham2597 Feb 18, 2024
d532402
AMP
haytham2597 Feb 18, 2024
0b839db
Add Print Modules Still in progress
haytham2597 Feb 19, 2024
98cabfa
Add some printing module
haytham2597 Feb 19, 2024
669b4fa
Fix some dotnet build. Need fix tests
haytham2597 Feb 20, 2024
3940414
Fast tensor accessor for ToArray()
haytham2597 Jun 30, 2024
3469d7a
Update local
haytham2597 Jun 30, 2024
5062339
fix local build dotnet
haytham2597 Jun 30, 2024
3a467af
Fast ToArray() TensorAccessor
haytham2597 Jul 2, 2024
18c7528
Fast tensor accesor
haytham2597 Jul 2, 2024
728c9fb
fix accesor for every types
haytham2597 Jul 9, 2024
a9a611a
GradScaler
haytham2597 Jul 12, 2024
4a406ec
Trying fix build for azure
haytham2597 Jul 14, 2024
280c8d5
Range sequential
haytham2597 Jul 17, 2024
3c42a87
AMPManager
haytham2597 Jul 19, 2024
7cd7f9c
Amp
haytham2597 Jul 20, 2024
1293483
update
haytham2597 Jul 20, 2024
0c2769a
fix azure devops?
haytham2597 Jul 21, 2024
eafdd1e
fix test?
haytham2597 Jul 21, 2024
c0883d9
fix mac test?
haytham2597 Jul 21, 2024
9ac78bd
AMP Problem outscope
haytham2597 Jul 24, 2024
d6a0c28
gradscale, device cuda properties, etc.
haytham2597 Sep 3, 2024
21ce055
some gradscaler. Need grad_scale and found_inf attr in optimizer
haytham2597 Sep 3, 2024
e9f34c8
Merge branch 'main' of https://github.com/dotnet/TorchSharp
haytham2597 Sep 3, 2024
c70b523
update v2.4.0
haytham2597 Sep 3, 2024
36b79b9
some advance
haytham2597 Sep 5, 2024
376f4fb
Improve autocastmode
haytham2597 Sep 8, 2024
9f4a48b
Some Autocast f16, f32
haytham2597 Oct 18, 2024
f84392b
fix test jit, it is literally close
haytham2597 Oct 18, 2024
197c1e4
Test and some improve on autocast
haytham2597 Oct 19, 2024
061ec44
cross between tensors, improve grad scaler and add normalize #1382
haytham2597 Oct 21, 2024
851a09e
GELU approximate #1368
haytham2597 Oct 21, 2024
16aba79
Device Properties #462
haytham2597 Oct 21, 2024
441bbdd
tensor backward function signature #1376
haytham2597 Oct 21, 2024
194a1f0
Half, Bfloat16
haytham2597 Oct 21, 2024
63da9c2
some fix THSCuda
haytham2597 Oct 25, 2024
ce679e2
fast copy tensor accessor
haytham2597 Oct 25, 2024
958a187
rollback sln
haytham2597 Oct 25, 2024
abe9990
Merge branch 'main' into fast_tensor_accesor
NiklasGustafsson Oct 25, 2024
0b20f13
Numel
haytham2597 Oct 25, 2024
7df8e46
Merge branch 'fast_tensor_accesor' of https://github.com/haytham2597/…
haytham2597 Oct 25, 2024
1aa1f25
original sln and fix some issue
haytham2597 Oct 26, 2024
572bc3e
some
haytham2597 Oct 28, 2024
2c33985
Test and fix some error
haytham2597 Nov 1, 2024
5a6240c
trying fix comp THSCuda
haytham2597 Nov 4, 2024
0d7a585
updage
haytham2597 Feb 14, 2025
e524239
custom libtorch fullpatch
haytham2597 Feb 15, 2025
8f35385
some update
haytham2597 Mar 26, 2025
05c7efb
imprv
haytham2597 Mar 26, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -273,3 +273,7 @@ packages/
/.idea
/test/TorchSharpTest/exportsd.py
.vscode/settings.json
/TestClear
TestClear/
/nuget.config
/src/Native/LibTorchSharp/third_party
7 changes: 5 additions & 2 deletions Directory.Build.props
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
<Import Project="build/Dependencies.props" />

<PropertyGroup>
<CustomLibTorchFullPath></CustomLibTorchFullPath>
<Configuration Condition="'$(Configuration)'==''">Debug</Configuration>
<Configurations>Debug;Release</Configurations>
<_DefaultArchitecture>$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString().ToLower())</_DefaultArchitecture>
Expand Down Expand Up @@ -92,7 +93,6 @@
<VersionPrefix>$(LibTorchPackageVersion)</VersionPrefix>
<VersionSuffix></VersionSuffix>
</PropertyGroup>

<PropertyGroup>
<!-- turned on/off manually in separate CI jobs -->
<SkipCuda Condition="'$(TargetOS)' == 'mac'">true</SkipCuda>
Expand Down Expand Up @@ -164,8 +164,11 @@
<DefineConstants>$(DefineContants);DEBUG</DefineConstants>
<Optimize>false</Optimize>
</PropertyGroup>
<PropertyGroup Condition="'$(NativeTargetArchitecture)' == 'x64' and '$(SkipCuda)' != 'true' and '$(SkipNative)' != 'true'">
<DefineConstants>$(DefineContants);CUDA_TOOLKIT_FOUND</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="$(Configuration.StartsWith('Release'))">
<Optimize>true</Optimize>
</PropertyGroup>

</Project>
</Project>
4 changes: 4 additions & 0 deletions nuget.config
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<repositoryPath>F:\NugetPackages</repositoryPath>
</configuration>
Original file line number Diff line number Diff line change
@@ -0,0 +1,224 @@
{
"format": 1,
"restore": {
"K:\\Proyects_Repos\\TorchSharp\\pkg\\FileRestitcher\\FileRestitcher.Tests\\FileRestitcher.Tests.csproj": {}
},
"projects": {
"K:\\Proyects_Repos\\TorchSharp\\pkg\\FileRestitcher\\FileRestitcher.Tests\\FileRestitcher.Tests.csproj": {
"version": "1.0.0",
"restore": {
"projectUniqueName": "K:\\Proyects_Repos\\TorchSharp\\pkg\\FileRestitcher\\FileRestitcher.Tests\\FileRestitcher.Tests.csproj",
"projectName": "FileRestitcher.Tests",
"projectPath": "K:\\Proyects_Repos\\TorchSharp\\pkg\\FileRestitcher\\FileRestitcher.Tests\\FileRestitcher.Tests.csproj",
"packagesPath": "C:\\Users\\Dimitri\\.nuget\\packages\\",
"outputPath": "K:\\Proyects_Repos\\TorchSharp\\pkg\\FileRestitcher\\FileRestitcher.Tests\\FileRestitcher.Tests.NupkgProj\\",
"projectStyle": "PackageReference",
"crossTargeting": true,
"fallbackFolders": [
"C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
],
"configFilePaths": [
"K:\\Proyects_Repos\\TorchSharp\\NuGet.Config",
"C:\\Users\\Dimitri\\AppData\\Roaming\\NuGet\\NuGet.Config",
"C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.FallbackLocation.config",
"C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.Offline.config"
],
"originalTargetFrameworks": [
"net472",
"netstandard2.0"
],
"sources": {
"C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages\\": {},
"https://api.nuget.org/v3/index.json": {}
},
"frameworks": {
"net472": {
"targetAlias": "net472",
"projectReferences": {
"K:\\Proyects_Repos\\TorchSharp\\pkg\\FileRestitcher\\FileRestitcher\\FileRestitcher.csproj": {
"projectPath": "K:\\Proyects_Repos\\TorchSharp\\pkg\\FileRestitcher\\FileRestitcher\\FileRestitcher.csproj"
}
}
},
"netstandard2.0": {
"targetAlias": "netstandard2.0",
"projectReferences": {
"K:\\Proyects_Repos\\TorchSharp\\pkg\\FileRestitcher\\FileRestitcher\\FileRestitcher.csproj": {
"projectPath": "K:\\Proyects_Repos\\TorchSharp\\pkg\\FileRestitcher\\FileRestitcher\\FileRestitcher.csproj"
}
}
}
},
"warningProperties": {
"warnAsError": [
"NU1605"
]
},
"restoreAuditProperties": {
"enableAudit": "true",
"auditLevel": "low",
"auditMode": "all"
},
"SdkAnalysisLevel": "9.0.100"
},
"frameworks": {
"net472": {
"targetAlias": "net472",
"dependencies": {
"Microsoft.NET.Test.Sdk": {
"suppressParent": "None",
"target": "Package",
"version": "[16.9.4, )"
},
"coverlet.collector": {
"include": "Runtime, Build, Native, ContentFiles, Analyzers, BuildTransitive",
"suppressParent": "All",
"target": "Package",
"version": "[3.0.2, )"
},
"xunit": {
"suppressParent": "None",
"target": "Package",
"version": "[2.4.2, )"
}
},
"runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\9.0.100\\RuntimeIdentifierGraph.json"
},
"netstandard2.0": {
"targetAlias": "netstandard2.0",
"dependencies": {
"Microsoft.NET.Test.Sdk": {
"suppressParent": "None",
"target": "Package",
"version": "[16.9.4, )"
},
"NETStandard.Library": {
"suppressParent": "All",
"target": "Package",
"version": "[2.0.3, )",
"autoReferenced": true
},
"coverlet.collector": {
"include": "Runtime, Build, Native, ContentFiles, Analyzers, BuildTransitive",
"suppressParent": "All",
"target": "Package",
"version": "[3.0.2, )"
},
"xunit": {
"suppressParent": "None",
"target": "Package",
"version": "[2.4.2, )"
}
},
"imports": [
"net461",
"net462",
"net47",
"net471",
"net472",
"net48",
"net481"
],
"assetTargetFallback": true,
"warn": true,
"runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\9.0.100\\RuntimeIdentifierGraph.json"
}
}
},
"K:\\Proyects_Repos\\TorchSharp\\pkg\\FileRestitcher\\FileRestitcher\\FileRestitcher.csproj": {
"version": "1.0.0",
"restore": {
"projectUniqueName": "K:\\Proyects_Repos\\TorchSharp\\pkg\\FileRestitcher\\FileRestitcher\\FileRestitcher.csproj",
"projectName": "FileRestitcher",
"projectPath": "K:\\Proyects_Repos\\TorchSharp\\pkg\\FileRestitcher\\FileRestitcher\\FileRestitcher.csproj",
"packagesPath": "C:\\Users\\Dimitri\\.nuget\\packages\\",
"outputPath": "K:\\Proyects_Repos\\TorchSharp\\pkg\\FileRestitcher\\FileRestitcher\\FileRestitcher.NupkgProj\\",
"projectStyle": "PackageReference",
"crossTargeting": true,
"fallbackFolders": [
"C:\\Program Files (x86)\\Microsoft Visual Studio\\Shared\\NuGetPackages"
],
"configFilePaths": [
"K:\\Proyects_Repos\\TorchSharp\\NuGet.Config",
"C:\\Users\\Dimitri\\AppData\\Roaming\\NuGet\\NuGet.Config",
"C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.FallbackLocation.config",
"C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.Offline.config"
],
"originalTargetFrameworks": [
"net6.0",
"netstandard2.0"
],
"sources": {
"C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages\\": {},
"https://api.nuget.org/v3/index.json": {}
},
"frameworks": {
"net6.0": {
"targetAlias": "net6.0",
"projectReferences": {}
},
"netstandard2.0": {
"targetAlias": "netstandard2.0",
"projectReferences": {}
}
},
"warningProperties": {
"warnAsError": [
"NU1605"
]
},
"restoreAuditProperties": {
"enableAudit": "true",
"auditLevel": "low",
"auditMode": "all"
},
"SdkAnalysisLevel": "9.0.100"
},
"frameworks": {
"net6.0": {
"targetAlias": "net6.0",
"imports": [
"net461",
"net462",
"net47",
"net471",
"net472",
"net48",
"net481"
],
"assetTargetFallback": true,
"warn": true,
"frameworkReferences": {
"Microsoft.NETCore.App": {
"privateAssets": "all"
}
},
"runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\9.0.100\\RuntimeIdentifierGraph.json"
},
"netstandard2.0": {
"targetAlias": "netstandard2.0",
"dependencies": {
"NETStandard.Library": {
"suppressParent": "All",
"target": "Package",
"version": "[2.0.3, )",
"autoReferenced": true
}
},
"imports": [
"net461",
"net462",
"net47",
"net471",
"net472",
"net48",
"net481"
],
"assetTargetFallback": true,
"warn": true,
"runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\9.0.100\\RuntimeIdentifierGraph.json"
}
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
<?xml version="1.0" encoding="utf-8" standalone="no"?>
<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
<RestoreSuccess Condition=" '$(RestoreSuccess)' == '' ">True</RestoreSuccess>
<RestoreTool Condition=" '$(RestoreTool)' == '' ">NuGet</RestoreTool>
<ProjectAssetsFile Condition=" '$(ProjectAssetsFile)' == '' ">$(MSBuildThisFileDirectory)project.assets.json</ProjectAssetsFile>
<NuGetPackageRoot Condition=" '$(NuGetPackageRoot)' == '' ">$(UserProfile)\.nuget\packages\</NuGetPackageRoot>
<NuGetPackageFolders Condition=" '$(NuGetPackageFolders)' == '' ">C:\Users\Dimitri\.nuget\packages\;C:\Program Files (x86)\Microsoft Visual Studio\Shared\NuGetPackages</NuGetPackageFolders>
<NuGetProjectStyle Condition=" '$(NuGetProjectStyle)' == '' ">PackageReference</NuGetProjectStyle>
<NuGetToolVersion Condition=" '$(NuGetToolVersion)' == '' ">6.12.0</NuGetToolVersion>
</PropertyGroup>
<ItemGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
<SourceRoot Include="C:\Users\Dimitri\.nuget\packages\" />
<SourceRoot Include="C:\Program Files (x86)\Microsoft Visual Studio\Shared\NuGetPackages\" />
</ItemGroup>
<ImportGroup Condition=" '$(TargetFramework)' == '' AND '$(ExcludeRestorePackageImports)' != 'true' ">
<Import Project="$(NuGetPackageRoot)xunit.core\2.4.2\buildMultiTargeting\xunit.core.props" Condition="Exists('$(NuGetPackageRoot)xunit.core\2.4.2\buildMultiTargeting\xunit.core.props')" />
<Import Project="$(NuGetPackageRoot)microsoft.net.test.sdk\16.9.4\buildMultiTargeting\Microsoft.NET.Test.Sdk.props" Condition="Exists('$(NuGetPackageRoot)microsoft.net.test.sdk\16.9.4\buildMultiTargeting\Microsoft.NET.Test.Sdk.props')" />
</ImportGroup>
<ImportGroup Condition=" '$(TargetFramework)' == 'net472' AND '$(ExcludeRestorePackageImports)' != 'true' ">
<Import Project="$(NuGetPackageRoot)xunit.core\2.4.2\build\xunit.core.props" Condition="Exists('$(NuGetPackageRoot)xunit.core\2.4.2\build\xunit.core.props')" />
<Import Project="$(NuGetPackageRoot)microsoft.codecoverage\16.9.4\build\netstandard1.0\Microsoft.CodeCoverage.props" Condition="Exists('$(NuGetPackageRoot)microsoft.codecoverage\16.9.4\build\netstandard1.0\Microsoft.CodeCoverage.props')" />
<Import Project="$(NuGetPackageRoot)microsoft.net.test.sdk\16.9.4\build\net45\Microsoft.NET.Test.Sdk.props" Condition="Exists('$(NuGetPackageRoot)microsoft.net.test.sdk\16.9.4\build\net45\Microsoft.NET.Test.Sdk.props')" />
</ImportGroup>
<ImportGroup Condition=" '$(TargetFramework)' == 'netstandard2.0' AND '$(ExcludeRestorePackageImports)' != 'true' ">
<Import Project="$(NuGetPackageRoot)xunit.core\2.4.2\build\xunit.core.props" Condition="Exists('$(NuGetPackageRoot)xunit.core\2.4.2\build\xunit.core.props')" />
<Import Project="$(NuGetPackageRoot)microsoft.codecoverage\16.9.4\build\netstandard1.0\Microsoft.CodeCoverage.props" Condition="Exists('$(NuGetPackageRoot)microsoft.codecoverage\16.9.4\build\netstandard1.0\Microsoft.CodeCoverage.props')" />
</ImportGroup>
<PropertyGroup Condition=" '$(TargetFramework)' == 'net472' AND '$(ExcludeRestorePackageImports)' != 'true' ">
<Pkgxunit_analyzers Condition=" '$(Pkgxunit_analyzers)' == '' ">C:\Users\Dimitri\.nuget\packages\xunit.analyzers\1.0.0</Pkgxunit_analyzers>
</PropertyGroup>
<PropertyGroup Condition=" '$(TargetFramework)' == 'netstandard2.0' AND '$(ExcludeRestorePackageImports)' != 'true' ">
<Pkgxunit_analyzers Condition=" '$(Pkgxunit_analyzers)' == '' ">C:\Users\Dimitri\.nuget\packages\xunit.analyzers\1.0.0</Pkgxunit_analyzers>
</PropertyGroup>
</Project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
<?xml version="1.0" encoding="utf-8" standalone="no"?>
<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ImportGroup Condition=" '$(TargetFramework)' == '' AND '$(ExcludeRestorePackageImports)' != 'true' ">
<Import Project="$(NuGetPackageRoot)xunit.core\2.4.2\buildMultiTargeting\xunit.core.targets" Condition="Exists('$(NuGetPackageRoot)xunit.core\2.4.2\buildMultiTargeting\xunit.core.targets')" />
</ImportGroup>
<ImportGroup Condition=" '$(TargetFramework)' == 'net472' AND '$(ExcludeRestorePackageImports)' != 'true' ">
<Import Project="$(NuGetPackageRoot)xunit.core\2.4.2\build\xunit.core.targets" Condition="Exists('$(NuGetPackageRoot)xunit.core\2.4.2\build\xunit.core.targets')" />
<Import Project="$(NuGetPackageRoot)microsoft.codecoverage\16.9.4\build\netstandard1.0\Microsoft.CodeCoverage.targets" Condition="Exists('$(NuGetPackageRoot)microsoft.codecoverage\16.9.4\build\netstandard1.0\Microsoft.CodeCoverage.targets')" />
<Import Project="$(NuGetPackageRoot)microsoft.net.test.sdk\16.9.4\build\net45\Microsoft.NET.Test.Sdk.targets" Condition="Exists('$(NuGetPackageRoot)microsoft.net.test.sdk\16.9.4\build\net45\Microsoft.NET.Test.Sdk.targets')" />
<Import Project="$(NuGetPackageRoot)coverlet.collector\3.0.2\build\netstandard1.0\coverlet.collector.targets" Condition="Exists('$(NuGetPackageRoot)coverlet.collector\3.0.2\build\netstandard1.0\coverlet.collector.targets')" />
</ImportGroup>
<ImportGroup Condition=" '$(TargetFramework)' == 'netstandard2.0' AND '$(ExcludeRestorePackageImports)' != 'true' ">
<Import Project="$(NuGetPackageRoot)netstandard.library\2.0.3\build\netstandard2.0\NETStandard.Library.targets" Condition="Exists('$(NuGetPackageRoot)netstandard.library\2.0.3\build\netstandard2.0\NETStandard.Library.targets')" />
<Import Project="$(NuGetPackageRoot)xunit.core\2.4.2\build\xunit.core.targets" Condition="Exists('$(NuGetPackageRoot)xunit.core\2.4.2\build\xunit.core.targets')" />
<Import Project="$(NuGetPackageRoot)microsoft.codecoverage\16.9.4\build\netstandard1.0\Microsoft.CodeCoverage.targets" Condition="Exists('$(NuGetPackageRoot)microsoft.codecoverage\16.9.4\build\netstandard1.0\Microsoft.CodeCoverage.targets')" />
<Import Project="$(NuGetPackageRoot)coverlet.collector\3.0.2\build\netstandard1.0\coverlet.collector.targets" Condition="Exists('$(NuGetPackageRoot)coverlet.collector\3.0.2\build\netstandard1.0\coverlet.collector.targets')" />
</ImportGroup>
</Project>
Loading