Skip to content

Commit c5cef31

Browse files
shauheenshauheen
authored
Merge pull request #1469 from shauheen/rc107
Merge master into release/preview for 0.7
2 parents fbcac80 + 4701dc6 commit c5cef31

File tree

1,001 files changed

+128060
-22361
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,001 files changed

+128060
-22361
lines changed

‎.gitmodules‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[submodule "src/Native/LIBMFNative/libmf"]
2+
path = src/Native/MatrixFactorizationNative/libmf
3+
url = https://github.com/cjlin1/libmf.git

‎.vsts-dotnet-ci.yml‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,35 @@
1+
################################################################################
2+
# ML.NET's PR validation build
3+
################################################################################
4+
5+
resources:
6+
containers:
7+
- container: LinuxContainer
8+
image: microsoft/dotnet-buildtools-prereqs:centos-7-b46d863-20180719033416
9+
110
phases:
211
- template: /build/ci/phase-template.yml
312
parameters:
413
name: Linux
514
buildScript: ./build.sh
6-
dockerImage: microsoft/dotnet-buildtools-prereqs:centos-7-b46d863-20180719033416
15+
queue:
16+
name: Hosted Ubuntu 1604
17+
container: LinuxContainer
18+
19+
- template: /build/ci/phase-template.yml
20+
parameters:
21+
name: Windows_x64
22+
buildScript: build.cmd
23+
queue:
24+
name: Hosted VS2017
725

826
- template: /build/ci/phase-template.yml
927
parameters:
10-
name: Windows_NT
28+
name: Windows_x86
29+
architecture: x86
1130
buildScript: build.cmd
1231
queue:
13-
name: DotNetCore-Windows
32+
name: Hosted VS2017
1433

1534
- template: /build/ci/phase-template.yml
1635
parameters:

‎Directory.Build.props‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,17 @@
66

77
<PropertyGroup>
88
<Configuration Condition="'$(Configuration)'==''">Debug</Configuration>
9+
<Configurations>Debug;Release;Debug-Intrinsics;Release-Intrinsics</Configurations>
910
<Platform Condition="'$(Platform)'==''">AnyCPU</Platform>
10-
<NativeTargetArchitecture Condition="'$(NativeTargetArchitecture)' == ''">x64</NativeTargetArchitecture>
11+
<TargetArchitecture Condition="'$(TargetArchitecture)' == ''">x64</TargetArchitecture>
12+
<NativeTargetArchitecture Condition="'$(NativeTargetArchitecture)' == ''">$(TargetArchitecture)</NativeTargetArchitecture>
1113
<PlatformConfig>$(Platform).$(Configuration)</PlatformConfig>
1214
</PropertyGroup>
1315

1416
<PropertyGroup>
1517
<RestoreSources>
1618
https://api.nuget.org/v3/index.json;
19+
https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json;
1720
https://dotnet.myget.org/F/dotnet-core/api/v3/index.json;
1821
</RestoreSources>
1922
</PropertyGroup>
@@ -106,6 +109,16 @@
106109
<UseIntrinsics Condition="'$(UseIntrinsics)' == ''">$(Configuration.EndsWith('-Intrinsics'))</UseIntrinsics>
107110
</PropertyGroup>
108111

112+
<!-- Need to explicitly set these properties for the -Intrinsics configurations becuase they are typically based off 'Debug' or 'Release' configs -->
113+
<!-- Taken from https://github.com/dotnet/sdk/blob/073c98b92c81066c6c2e17c3674adbb6e833409a/src/Tasks/Microsoft.NET.Build.Tasks/targets/Microsoft.NET.Sdk.props#L41-L47 -->
114+
<PropertyGroup Condition="'$(Configuration)' == 'Debug-Intrinsics'">
115+
<DebugSymbols>true</DebugSymbols>
116+
<Optimize>false</Optimize>
117+
</PropertyGroup>
118+
<PropertyGroup Condition="'$(Configuration)' == 'Release-Intrinsics'">
119+
<Optimize>true</Optimize>
120+
</PropertyGroup>
121+
109122
<PropertyGroup>
110123
<CustomAfterMicrosoftCommonTargets>$(RepoRoot)build\AfterCommonTargets.targets</CustomAfterMicrosoftCommonTargets>
111124
</PropertyGroup>

‎Microsoft.ML.sln‎

Lines changed: 225 additions & 157 deletions
Large diffs are not rendered by default.

‎README.md‎

Lines changed: 30 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -4,23 +4,23 @@
44

55
[ML.NET](https://www.microsoft.com/net/learn/apps/machine-learning-and-ai/ml-dotnet) is a cross-platform open-source machine learning framework which makes machine learning accessible to .NET developers.
66

7-
ML.NET allows .NET developers to develop their own models and infuse custom ML into their applications without prior expertise in developing or tuning machine learning models, all in .NET.
7+
ML.NET allows .NET developers to develop their own models and infuse custom machine learning into their applications, using .NET, even without prior expertise in developing or tuning machine learning models.
88

9-
ML.NET was originally developed in Microsoft Research and evolved into a significant framework over the last decade and is used across many product groups in Microsoft like Windows, Bing, PowerPoint, Excel and more.
9+
ML.NET was originally developed in Microsoft Research, and evolved into a significant framework over the last decade and is used across many product groups in Microsoft like Windows, Bing, PowerPoint, Excel and more.
1010

11-
With this first preview release ML.NET enables ML tasks like classification (e.g. support text classification, sentiment analysis) and regression (e.g. price-prediction).
11+
ML.NET enables machine learning tasks like classification (for example: support text classification, sentiment analysis) and regression (for example, price-prediction).
1212

13-
Along with these ML capabilities this first release of ML.NET also brings the first draft of .NET APIs for training models, using models for predictions, as well as the core components of this framework such as learning algorithms, transforms, and ML data structures.
13+
Along with these ML capabilities, this first release of ML.NET also brings the first draft of .NET APIs for training models, using models for predictions, as well as the core components of this framework such as learning algorithms, transforms, and ML data structures.
1414

1515
## Installation
1616

1717
[![NuGet Status](https://img.shields.io/nuget/v/Microsoft.ML.svg?style=flat)](https://www.nuget.org/packages/Microsoft.ML/)
1818

19-
ML.NET runs on Windows, Linux, and macOS - any platform where 64 bit [.NET Core](https://github.com/dotnet/core) or later is available.
19+
ML.NET runs on Windows, Linux, and macOS - any platform where x64 [.NET Core](https://github.com/dotnet/core) or later is available. In addition, .NET Framework on Windows x64 is also supported.
2020

21-
The current release is 0.5. Check out the [release notes](docs/release-notes/0.5/release-0.5.md).
21+
The current release is 0.6. Check out the [release notes](docs/release-notes/0.6/release-0.6.md) to see what's new.
2222

23-
First ensure you have installed [.NET Core 2.0](https://www.microsoft.com/net/learn/get-started) or later. ML.NET also works on the .NET Framework. Note that ML.NET currently must run in a 64 bit process.
23+
First, ensure you have installed [.NET Core 2.0](https://www.microsoft.com/net/learn/get-started) or later. ML.NET also works on the .NET Framework. Note that ML.NET currently must run in a 64-bit process.
2424

2525
Once you have an app, you can install the ML.NET NuGet package from the .NET Core CLI using:
2626
```
@@ -32,9 +32,9 @@ or from the NuGet package manager:
3232
Install-Package Microsoft.ML
3333
```
3434

35-
Or alternatively you can add the Microsoft.ML package from within Visual Studio's NuGet package manager or via [Paket](https://github.com/fsprojects/Paket).
35+
Or alternatively, you can add the Microsoft.ML package from within Visual Studio's NuGet package manager or via [Paket](https://github.com/fsprojects/Paket).
3636

37-
Daily NuGet builds of the project are also available in our MyGet feed:
37+
Daily NuGet builds of the project are also available in our [MyGet](https://dotnet.myget.org/feed/dotnet-core/package/nuget/Microsoft.ML) feed:
3838

3939
> [https://dotnet.myget.org/F/dotnet-core/api/v3/index.json](https://dotnet.myget.org/F/dotnet-core/api/v3/index.json)
4040
@@ -62,28 +62,38 @@ For more information, see the [.NET Foundation Code of Conduct](https://dotnetfo
6262
## Examples
6363

6464
Here's an example of code to train a model to predict sentiment from text samples.
65-
(You can see the complete sample [here](test/Microsoft.ML.Tests/Scenarios/SentimentPredictionTests.cs)):
65+
(You can find a sample of the legacy API [here](test/Microsoft.ML.Tests/Scenarios/SentimentPredictionTests.cs)):
6666

6767
```C#
68-
var pipeline = new LearningPipeline();
69-
pipeline.Add(new TextLoader(dataPath).CreateFrom<SentimentData>(separator: ','));
70-
pipeline.Add(new TextFeaturizer("Features", "SentimentText"));
71-
pipeline.Add(new FastTreeBinaryClassifier());
72-
var model = pipeline.Train<SentimentData, SentimentPrediction>();
68+
var env = new LocalEnvironment();
69+
var reader = TextLoader.CreateReader(env, ctx => (
70+
Target: ctx.LoadFloat(2),
71+
FeatureVector: ctx.LoadFloat(3, 6)),
72+
separator: ',',
73+
hasHeader: true);
74+
var data = reader.Read(new MultiFileSource(dataPath));
75+
var classification = new MulticlassClassificationContext(env);
76+
var learningPipeline = reader.MakeNewEstimator()
77+
.Append(r => (
78+
r.Target,
79+
Prediction: classification.Trainers.Sdca(r.Target.ToKey(), r.FeatureVector)));
80+
var model = learningPipeline.Fit(data);
81+
7382
```
7483

7584
Now from the model we can make inferences (predictions):
7685

7786
```C#
78-
SentimentData data = new SentimentData
87+
var predictionFunc = model.MakePredictionFunction<SentimentInput, SentimentPrediction>(env);
88+
var prediction = predictionFunc.Predict(new SentimentData
7989
{
8090
SentimentText = "Today is a great day!"
81-
};
82-
83-
SentimentPrediction prediction = model.Predict(data);
84-
91+
});
8592
Console.WriteLine("prediction: " + prediction.Sentiment);
8693
```
94+
A cookbook that shows how to use these APIs for a variety of existing and new scenarios can be found [here](docs/code/MlNetCookBook.md).
95+
96+
8797
## Samples
8898

8999
We have a [repo of samples](https://github.com/dotnet/machinelearning-samples) that you can look at.

‎ROADMAP.md‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,12 @@ In the meanwhile, we are looking for contributions. An easy place to start is t
2323
* Native Binary high-performance format
2424

2525
### Featurization Improvements
26+
We already provide text/NLP and image processing functionalities that will be expanded
2627
* Text (*)
27-
* Natural language text preprocessing such as tokenization, part-of-speech tagging, and sentence breaking
28-
* Pre-trained text models that can be used for extracting of semantic or sentiment features from text
28+
* Natural language text preprocessing such as improving tokenization features, adding part-of-speech tagging, and sentence boundary disambiguation
29+
* Pre-trained text models (beyond current n-gram and pre-trained WordEmbedding text handling) that can further improve the extraction of semantic or sentiment features from text
2930
* Image (*)
30-
* Image preprocessing such as loading, resizing, and normalization if images
31+
* Image preprocessing such as loading, resizing, and normalization of images
3132
* Image featurization, including industry-standard pre-trained ImageNet neural models, such as ResNet and AlexNet
3233

3334
### Trained Model Management

‎THIRD-PARTY-NOTICES.TXT‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
ML.NET uses third-party libraries or other resources that may be
2+
distributed under licenses different than the ML.NET software.
3+
4+
In the event that we accidentally failed to list a required notice, please
5+
bring it to our attention. Post an issue or email us:
6+
7+
dotnet@microsoft.com
8+
9+
The attached notices are provided for information only.
10+
11+
License notice for LIBMF
12+
------------------------
13+
14+
https://github.com/cjlin1/libmf
15+
16+
Copyright (c) 2014-2015 The LIBMF Project.
17+
All rights reserved.
18+
19+
Redistribution and use in source and binary forms, with or without
20+
modification, are permitted provided that the following conditions
21+
are met:
22+
23+
1. Redistributions of source code must retain the above copyright
24+
notice, this list of conditions and the following disclaimer.
25+
26+
2. Redistributions in binary form must reproduce the above copyright
27+
notice, this list of conditions and the following disclaimer in the
28+
documentation and/or other materials provided with the distribution.
29+
30+
3. Neither name of copyright holders nor the names of its contributors
31+
may be used to endorse or promote products derived from this software
32+
without specific prior written permission.
33+
34+
35+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
36+
``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
37+
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
38+
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR
39+
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
40+
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
41+
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
42+
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
43+
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
44+
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
45+
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

‎build/BranchInfo.props‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<Project>
22
<PropertyGroup>
33
<MajorVersion>0</MajorVersion>
4-
<MinorVersion>6</MinorVersion>
4+
<MinorVersion>7</MinorVersion>
55
<PatchVersion>0</PatchVersion>
66
<PreReleaseLabel>preview</PreReleaseLabel>
77
</PropertyGroup>

‎build/Dependencies.props‎

Lines changed: 33 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,45 @@
11
<Project>
2+
3+
<!-- Core Product Dependencies -->
24
<PropertyGroup>
3-
<GoogleProtobufPackageVersion>3.5.1</GoogleProtobufPackageVersion>
45
<NewtonsoftJsonPackageVersion>10.0.3</NewtonsoftJsonPackageVersion>
5-
<ParquetDotNetPackageVersion>2.1.3</ParquetDotNetPackageVersion>
6-
<SystemThreadingTasksDataflowPackageVersion>4.8.0</SystemThreadingTasksDataflowPackageVersion>
76
<SystemCodeDomPackageVersion>4.4.0</SystemCodeDomPackageVersion>
7+
<SystemCollectionsImmutableVersion>1.5.0</SystemCollectionsImmutableVersion>
8+
<SystemMemoryVersion>4.5.1</SystemMemoryVersion>
89
<SystemReflectionEmitLightweightPackageVersion>4.3.0</SystemReflectionEmitLightweightPackageVersion>
9-
<PublishSymbolsPackageVersion>1.0.0-beta-62824-02</PublishSymbolsPackageVersion>
10-
<LightGBMPackageVersion>2.1.2.2</LightGBMPackageVersion>
11-
<MlNetMklDepsPackageVersion>0.0.0.6</MlNetMklDepsPackageVersion>
10+
<SystemThreadingTasksDataflowPackageVersion>4.8.0</SystemThreadingTasksDataflowPackageVersion>
11+
<SystemComponentModelCompositionVersion>4.5.0</SystemComponentModelCompositionVersion>
12+
</PropertyGroup>
13+
14+
<!-- Other/Non-Core Product Dependencies -->
15+
<PropertyGroup>
16+
<GoogleProtobufPackageVersion>3.5.1</GoogleProtobufPackageVersion>
17+
<LightGBMPackageVersion>2.2.1.1</LightGBMPackageVersion>
18+
<MicrosoftMLScoring>1.1.0</MicrosoftMLScoring>
19+
<MlNetMklDepsPackageVersion>0.0.0.7</MlNetMklDepsPackageVersion>
20+
<ParquetDotNetPackageVersion>2.1.3</ParquetDotNetPackageVersion>
1221
<SystemDrawingCommonPackageVersion>4.5.0</SystemDrawingCommonPackageVersion>
13-
<BenchmarkDotNetVersion>0.11.1</BenchmarkDotNetVersion>
22+
<SystemIOFileSystemAccessControl>4.5.0</SystemIOFileSystemAccessControl>
23+
<SystemSecurityPrincipalWindows>4.5.0</SystemSecurityPrincipalWindows>
1424
<TensorFlowVersion>1.10.0</TensorFlowVersion>
15-
<SystemCollectionsImmutableVersion>1.5.0</SystemCollectionsImmutableVersion>
16-
<SystemMemoryVersion>4.5.1</SystemMemoryVersion>
25+
</PropertyGroup>
26+
27+
<!-- Code Analyzer Dependencies -->
28+
<PropertyGroup>
1729
<MicrosoftCodeAnalysisCSharpVersion>2.9.0</MicrosoftCodeAnalysisCSharpVersion>
1830
<MicrosoftCSharpVersion>4.5.0</MicrosoftCSharpVersion>
1931
<SystemCompositionVersion>1.2.0</SystemCompositionVersion>
20-
<MicrosoftMLScoring>1.1.0</MicrosoftMLScoring>
21-
<SystemIOFileSystemAccessControl>4.5.0</SystemIOFileSystemAccessControl>
22-
<SystemSecurityPrincipalWindows>4.5.0</SystemSecurityPrincipalWindows>
2332
</PropertyGroup>
33+
34+
<!-- Build/infrastructure Dependencies -->
35+
<PropertyGroup>
36+
<PublishSymbolsPackageVersion>1.0.0-beta-62824-02</PublishSymbolsPackageVersion>
37+
</PropertyGroup>
38+
39+
<!-- Test-only Dependencies -->
40+
<PropertyGroup>
41+
<BenchmarkDotNetVersion>0.11.1</BenchmarkDotNetVersion>
42+
<MicrosoftMLTestModelsPackageVersion>0.0.3-test</MicrosoftMLTestModelsPackageVersion>
43+
</PropertyGroup>
44+
2445
</Project>

‎build/ci/phase-template.yml‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,15 @@
11
parameters:
22
name: ''
3+
architecture: x64
34
buildScript: ''
4-
dockerImage: ''
55
queue: {}
66

77
phases:
88
- phase: ${{ parameters.name }}
99
variables:
1010
_buildScript: ${{ parameters.buildScript }}
1111
_phaseName: ${{ parameters.name }}
12-
# if dockerImage is not equal to '' then run under docker container
13-
${{ if ne(parameters.dockerImage, '') }}:
14-
_PREVIEW_VSTS_DOCKER_IMAGE: ${{ parameters.dockerImage }}
12+
_arch: ${{ parameters.architecture }}
1513
queue:
1614
parallel: 99
1715
matrix:
@@ -21,8 +19,11 @@ phases:
2119
_configuration: Release
2220
${{ insert }}: ${{ parameters.queue }}
2321
steps:
24-
- script: $(_buildScript) -$(_configuration)
22+
- script: $(_buildScript) -$(_configuration) -buildArch=$(_arch)
2523
displayName: Build
24+
- ${{ if eq(parameters.name, 'MacOS') }}:
25+
- script: brew install libomp
26+
displayName: Install OpenMP runtime dependency
2627
- script: $(_buildScript) -$(_configuration) -runtests
2728
displayName: Run Tests
2829
- task: PublishTestResults@2

0 commit comments

Comments
 (0)